def test_fresh_bootstrap(self): with temporary_dir() as fresh_bootstrap_dir: self.set_bootstrap_options(pants_bootstrapdir=fresh_bootstrap_dir) # Initialize the Ivy subsystem self.context() Bootstrapper.default_ivy() bootstrap_jar_path = os.path.join(fresh_bootstrap_dir, 'tools', 'jvm', 'ivy', 'bootstrap.jar') self.assertTrue(os.path.exists(bootstrap_jar_path))
def publish(self, ivyxml_path, jar, entry, repo, published): """Run ivy to publish a jar. ivyxml_path is the path to the ivy file; published is a list of jars published so far (including this one). entry is a pushdb entry.""" jvm_args = self._ivy_jvm_args(repo) resolver = repo['resolver'] path = repo.get('path') try: ivy = Bootstrapper.default_ivy() except Bootstrapper.Error as e: raise TaskError('Failed to push {0}! {1}'.format(pushdb_coordinate(jar, entry), e)) ivysettings = self.generate_ivysettings(ivy, published, publish_local=path) args = [ '-settings', ivysettings, '-ivy', ivyxml_path, '-deliverto', '%s/[organisation]/[module]/ivy-[revision].xml' % self.workdir, '-publish', resolver, '-publishpattern', '%s/[organisation]/[module]/' '[artifact]-[revision](-[classifier]).[ext]' % self.workdir, '-revision', entry.version().version(), '-m2compatible', ] if LogOptions.stderr_log_level() == logging.DEBUG: args.append('-verbose') if self.local_snapshot: args.append('-overwrite') try: ivy.execute(jvm_options=jvm_args, args=args, workunit_factory=self.context.new_workunit, workunit_name='jar-publish') except Ivy.Error as e: raise TaskError('Failed to push {0}! {1}'.format(pushdb_coordinate(jar, entry), e))
def exec_ivy(cls, ivy, confs, ivyxml, args, jvm_options, executor, workunit_name, workunit_factory): """ :API: public """ ivy = ivy or Bootstrapper.default_ivy() ivy_args = ['-ivy', ivyxml] ivy_args.append('-confs') ivy_args.extend(confs) ivy_args.extend(args) ivy_jvm_options = list(jvm_options) # Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng. ivy_jvm_options.append('-Dsun.io.useCanonCaches=false') runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor) try: with ivy.resolution_lock: result = execute_runner(runner, workunit_factory=workunit_factory, workunit_name=workunit_name) if result != 0: raise IvyUtils.IvyError('Ivy returned {result}. cmd={cmd}'.format(result=result, cmd=runner.cmd)) except runner.executor.Error as e: raise IvyUtils.IvyError(e)
def exec_ivy(cls, ivy, confs, ivyxml, args, jvm_options, executor, workunit_name, workunit_factory): ivy = ivy or Bootstrapper.default_ivy() ivy_args = ['-ivy', ivyxml] ivy_args.append('-confs') ivy_args.extend(confs) ivy_args.extend(args) ivy_jvm_options = list(jvm_options) # Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng. ivy_jvm_options.append('-Dsun.io.useCanonCaches=false') runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor) try: result = execute_runner(runner, workunit_factory=workunit_factory, workunit_name=workunit_name) if result != 0: raise IvyUtils.IvyError( 'Ivy returned {result}. cmd={cmd}'.format(result=result, cmd=runner.cmd)) except runner.executor.Error as e: raise IvyUtils.IvyError(e)
def publish(ivyxml_path): ivysettings = self.generate_ivysettings(published, publish_local=path) args = [ '-settings', ivysettings, '-ivy', ivyxml_path, '-deliverto', '%s/[organisation]/[module]/ivy-[revision].xml' % self.workdir, '-publish', resolver, '-publishpattern', '%s/[organisation]/[module]/' '[artifact]-[revision](-[classifier]).[ext]' % self.workdir, '-revision', newver.version(), '-m2compatible', ] if LogOptions.stderr_log_level() == logging.DEBUG: args.append('-verbose') if self.snapshot: args.append('-overwrite') try: ivy = Bootstrapper.default_ivy() ivy.execute(jvm_options=jvm_args, args=args, workunit_factory=self.context.new_workunit, workunit_name='jar-publish') except (Bootstrapper.Error, Ivy.Error) as e: raise TaskError('Failed to push %s! %s' % (jar_coordinate(jar, newver.version()), e))
def publish(ivyxml_path): try: ivy = Bootstrapper.default_ivy() except Bootstrapper.Error as e: raise TaskError('Failed to push %s! %s' % (jar_coordinate(jar, newver.version()), e)) ivysettings = self.generate_ivysettings(ivy, published, publish_local=path) args = [ '-settings', ivysettings, '-ivy', ivyxml_path, '-deliverto', '%s/[organisation]/[module]/ivy-[revision].xml' % self.workdir, '-publish', resolver, '-publishpattern', '%s/[organisation]/[module]/' '[artifact]-[revision](-[classifier]).[ext]' % self.workdir, '-revision', newver.version(), '-m2compatible', ] if LogOptions.stderr_log_level() == logging.DEBUG: args.append('-verbose') if self.snapshot: args.append('-overwrite') try: ivy.execute(jvm_options=jvm_args, args=args, workunit_factory=self.context.new_workunit, workunit_name='jar-publish') except Ivy.Error as e: raise TaskError('Failed to push %s! %s' % (jar_coordinate(jar, newver.version()), e))
def execute_junit_runner(self, content): # Create the temporary base test directory test_rel_path = 'tests/java/org/pantsbuild/foo' test_abs_path = os.path.join(self.build_root, test_rel_path) self.create_dir(test_rel_path) # Generate the temporary java test source code. test_java_file_rel_path = os.path.join(test_rel_path, 'FooTest.java') test_java_file_abs_path = os.path.join(self.build_root, test_java_file_rel_path) self.create_file(test_java_file_rel_path, content) # Invoke ivy to resolve classpath for junit. distribution = Distribution.cached(jdk=True) executor = SubprocessExecutor(distribution=distribution) classpath_file_abs_path = os.path.join(test_abs_path, 'junit.classpath') ivy = Bootstrapper.default_ivy() ivy.execute(args=['-cachepath', classpath_file_abs_path, '-dependency', 'junit', 'junit-dep', '4.10'], executor=executor) with open(classpath_file_abs_path) as fp: classpath = fp.read() # Now directly invoking javac to compile the test java code into java class # so later we can inject the class into products mapping for JUnitRun to execute # the test on. javac = distribution.binary('javac') subprocess.check_call( [javac, '-d', test_abs_path, '-cp', classpath, test_java_file_abs_path]) # Create a java_tests target and a synthetic resource target. java_tests = self.create_library(test_rel_path, 'java_tests', 'foo_test', ['FooTest.java']) resources = self.make_target('some_resources', Resources) # Set the context with the two targets, one java_tests target and # one synthetic resources target. # The synthetic resources target is to make sure we won't regress # in the future with bug like https://github.com/pantsbuild/pants/issues/508. Note # in that bug, the resources target must be the first one in the list. context = self.context(target_roots=[resources, java_tests]) # Before we run the task, we need to inject the "classes_by_target" with # the compiled test java classes that JUnitRun will know which test # classes to execute. In a normal run, this "classes_by_target" will be # populated by java compiling step. class_products = context.products.get_data( 'classes_by_target', lambda: defaultdict(MultipleRootedProducts)) java_tests_products = MultipleRootedProducts() java_tests_products.add_rel_paths(test_abs_path, ['FooTest.class']) class_products[java_tests] = java_tests_products # Also we need to add the FooTest.class's classpath to the compile_classpath # products data mapping so JUnitRun will be able to add that into the final # classpath under which the junit will be executed. self.populate_compile_classpath( context=context, classpath=[test_abs_path]) # Finally execute the task. self.execute(context)
def exec_ivy(self, target_workdir, targets, args, confs=None, ivy=None, workunit_name='ivy', workunit_factory=None, symlink_ivyxml=False, jars=None): ivy = ivy or Bootstrapper.default_ivy() if not isinstance(ivy, Ivy): raise ValueError('The ivy argument supplied must be an Ivy instance, given %s of type %s' % (ivy, type(ivy))) ivyxml = os.path.join(target_workdir, 'ivy.xml') if not jars: jars, excludes = self._calculate_classpath(targets) else: excludes = set() ivy_args = ['-ivy', ivyxml] confs_to_resolve = confs or ['default'] ivy_args.append('-confs') ivy_args.extend(confs_to_resolve) ivy_args.extend(args) if not self._transitive: ivy_args.append('-notransitive') ivy_args.extend(self._args) def safe_link(src, dest): try: os.unlink(dest) except OSError as e: if e.errno != errno.ENOENT: raise os.symlink(src, dest) with IvyUtils.ivy_lock: self._generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve) runner = ivy.runner(jvm_options=self._jvm_options, args=ivy_args) try: result = util.execute_runner(runner, workunit_factory=workunit_factory, workunit_name=workunit_name) # Symlink to the current ivy.xml file (useful for IDEs that read it). if symlink_ivyxml: ivyxml_symlink = os.path.join(self._workdir, 'ivy.xml') safe_link(ivyxml, ivyxml_symlink) if result != 0: raise TaskError('Ivy returned %d' % result) except runner.executor.Error as e: raise TaskError(e)
def publish(self, publications, jar, entry, repo, published): """Run ivy to publish a jar. ivyxml_path is the path to the ivy file; published is a list of jars published so far (including this one). entry is a pushdb entry.""" try: ivy = Bootstrapper.default_ivy() except Bootstrapper.Error as e: raise TaskError('Failed to push {0}! {1}'.format( pushdb_coordinate(jar, entry), e)) path = repo.get('path') ivysettings = self.generate_ivysettings(ivy, published, publish_local=path) version = entry.version().version() ivyxml = self.generate_ivy(jar, version, publications) resolver = repo['resolver'] args = [ '-settings', ivysettings, '-ivy', ivyxml, # Without this setting, the ivy.xml is delivered to the CWD, littering the workspace. We # don't need the ivy.xml, so just give it path under the workdir we won't use. '-deliverto', ivyxml + '.unused', '-publish', resolver, '-publishpattern', '{}/[organisation]/[module]/' '[artifact]-[revision](-[classifier]).[ext]'.format(self.workdir), '-revision', version, '-m2compatible', ] # TODO(John Sirois): global logging options should be hidden behind some sort of log manager # that we can: # a.) obtain a handle to (dependency injection or manual plumbing) # b.) query for log detail, ie: `if log_manager.is_verbose:` if self.get_options().level == 'debug': args.append('-verbose') if self.local_snapshot: args.append('-overwrite') try: jvm_options = self._ivy_jvm_options(repo) ivy.execute(jvm_options=jvm_options, args=args, workunit_factory=self.context.new_workunit, workunit_name='ivy-publish') except Ivy.Error as e: raise TaskError('Failed to push {0}! {1}'.format( pushdb_coordinate(jar, entry), e))
def publish(self, publications, jar, entry, repo, published): """Run ivy to publish a jar. ivyxml_path is the path to the ivy file; published is a list of jars published so far (including this one). entry is a pushdb entry.""" try: ivy = Bootstrapper.default_ivy() except Bootstrapper.Error as e: raise TaskError("Failed to push {0}! {1}".format(pushdb_coordinate(jar, entry), e)) path = repo.get("path") ivysettings = self.generate_ivysettings(ivy, published, publish_local=path) version = entry.version().version() ivyxml = self.generate_ivy(jar, version, publications) resolver = repo["resolver"] args = [ "-settings", ivysettings, "-ivy", ivyxml, # Without this setting, the ivy.xml is delivered to the CWD, littering the workspace. We # don't need the ivy.xml, so just give it path under the workdir we won't use. "-deliverto", ivyxml + ".unused", "-publish", resolver, "-publishpattern", "{}/[organisation]/[module]/" "[artifact]-[revision](-[classifier]).[ext]".format(self.workdir), "-revision", version, "-m2compatible", ] # TODO(John Sirois): global logging options should be hidden behind some sort of log manager # that we can: # a.) obtain a handle to (dependency injection or manual plumbing) # b.) query for log detail, ie: `if log_manager.is_verbose:` if self.get_options().level == "debug": args.append("-verbose") if self.local_snapshot: args.append("-overwrite") try: jvm_options = self._ivy_jvm_options(repo) ivy.execute( jvm_options=jvm_options, args=args, workunit_factory=self.context.new_workunit, workunit_name="ivy-publish", ) except Ivy.Error as e: raise TaskError("Failed to push {0}! {1}".format(pushdb_coordinate(jar, entry), e))
def mapjars(self, genmap, target, executor, workunit_factory=None, jars=None): """Resolves jars for the target and stores their locations in genmap. :param genmap: The jar_dependencies ProductMapping entry for the required products. :param target: The target whose jar dependencies are being retrieved. :param jars: If specified, resolves the given jars rather than :type jars: List of :class:`pants.backend.jvm.targets.jar_dependency.JarDependency` (jar()) objects. """ mapdir = os.path.join(self.mapto_dir(), target.id) safe_mkdir(mapdir, clean=True) ivyargs = [ '-retrieve', '%s/[organisation]/[artifact]/[conf]/' '[organisation]-[artifact]-[revision](-[classifier]).[ext]' % mapdir, '-symlink', ] self.exec_ivy(mapdir, [target], ivyargs, confs=target.payload.configurations, ivy=Bootstrapper.default_ivy(executor), workunit_factory=workunit_factory, workunit_name='map-jars', jars=jars) for org in os.listdir(mapdir): orgdir = os.path.join(mapdir, org) if os.path.isdir(orgdir): for name in os.listdir(orgdir): artifactdir = os.path.join(orgdir, name) if os.path.isdir(artifactdir): for conf in os.listdir(artifactdir): confdir = os.path.join(artifactdir, conf) for f in os.listdir(confdir): if self.is_mappable_artifact(org, name, f): # TODO(John Sirois): kill the org and (org, name) exclude mappings in favor of a # conf whitelist genmap.add(org, confdir).append(f) genmap.add((org, name), confdir).append(f) genmap.add(target, confdir).append(f) genmap.add((target, conf), confdir).append(f) genmap.add((org, name, conf), confdir).append(f)
def publish(self, publications, jar, version, repo, published): """Run ivy to publish a jar. :param str ivyxml_path: The path to the ivy file. :param list published: A list of jars published so far (including this one). """ try: ivy = Bootstrapper.default_ivy() except Bootstrapper.Error as e: raise TaskError('Failed to push {0}! {1}'.format(jar, e)) if ivy.ivy_settings is None: raise TaskError('An ivysettings.xml with writeable resolvers is required for publishing, ' 'but none was configured.') path = repo.get('path') ivysettings = self.generate_ivysettings(self.fetch_ivysettings(ivy), published, publish_local=path) ivyxml = self.generate_ivy(jar, version, publications) resolver = repo['resolver'] args = [ '-settings', ivysettings, '-ivy', ivyxml, # Without this setting, the ivy.xml is delivered to the CWD, littering the workspace. We # don't need the ivy.xml, so just give it path under the workdir we won't use. '-deliverto', ivyxml + '.unused', '-publish', resolver, '-publishpattern', '{}/[organisation]/[module]/' '[artifact]-[revision](-[classifier]).[ext]'.format(self.workdir), '-revision', version, '-m2compatible', ] if self.get_options().level == 'debug': args.append('-verbose') if self.get_options().local: args.append('-overwrite') if not self.get_options().dryrun: try: ivy.execute(jvm_options=self._ivy_jvm_options(repo), args=args, workunit_factory=self.context.new_workunit, workunit_name='ivy-publish') except Ivy.Error as e: raise TaskError('Failed to push {0}! {1}'.format(jar, e)) else: print("\nDRYRUN- would have pushed:{} using {} resolver.".format(self.jar_coordinate(jar, version), resolver))
def _exec_ivy(self, target_workdir, targets, args, executor=None, confs=None, ivy=None, workunit_name='ivy', use_soft_excludes=False, resolve_hash_name=None): ivy_jvm_options = self.get_options().jvm_options[:] # Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng. ivy_jvm_options.append('-Dsun.io.useCanonCaches=false') ivy = ivy or Bootstrapper.default_ivy() ivyxml = os.path.join(target_workdir, 'ivy.xml') ivy_args = ['-ivy', ivyxml] confs_to_resolve = confs or ('default', ) ivy_args.append('-confs') ivy_args.extend(confs_to_resolve) ivy_args.extend(args) # TODO(John Sirois): merge the code below into IvyUtils or up here; either way, better # diagnostics can be had in `IvyUtils.generate_ivy` if this is done. # See: https://github.com/pantsbuild/pants/issues/2239 try: jars, excludes = IvyUtils.calculate_classpath( targets, gather_excludes=not use_soft_excludes) with IvyUtils.ivy_lock: IvyUtils.generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve, resolve_hash_name) runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor) try: result = execute_runner( runner, workunit_factory=self.context.new_workunit, workunit_name=workunit_name) if result != 0: raise self.Error( 'Ivy returned {result}. cmd={cmd}'.format( result=result, cmd=runner.cmd)) except runner.executor.Error as e: raise self.Error(e) except IvyUtils.IvyError as e: raise self.Error('Failed to prepare ivy resolve: {}'.format(e))
def exec_ivy(self, target_workdir, targets, args, executor=None, confs=None, ivy=None, workunit_name='ivy', jars=None, use_soft_excludes=False, resolve_hash_name=None): ivy_jvm_options = copy.copy(self.get_options().jvm_options) # Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng. ivy_jvm_options.append('-Dsun.io.useCanonCaches=false') ivy = ivy or Bootstrapper.default_ivy() ivyxml = os.path.join(target_workdir, 'ivy.xml') if not jars: jars, excludes = IvyUtils.calculate_classpath( targets, gather_excludes=not use_soft_excludes) else: excludes = set() ivy_args = ['-ivy', ivyxml] confs_to_resolve = confs or ['default'] ivy_args.append('-confs') ivy_args.extend(confs_to_resolve) ivy_args.extend(args) with IvyUtils.ivy_lock: IvyUtils.generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve, resolve_hash_name) runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor) try: result = execute_runner( runner, workunit_factory=self.context.new_workunit, workunit_name=workunit_name) if result != 0: raise TaskError('Ivy returned {result}. cmd={cmd}'.format( result=result, cmd=runner.cmd)) except runner.executor.Error as e: raise TaskError(e)
def run_antlrs(self, output_dir): args = [ '-dependency', 'org.antlr', 'antlr', self.target.antlr_version, '-types', 'jar', '-main', 'org.antlr.Tool', '--', '-fo', output_dir ] for source in self.target.sources_relative_to_buildroot(): abs_path = os.path.join(get_buildroot(), source) args.append(abs_path) try: ivy = Bootstrapper.default_ivy() ivy.execute( args=args ) # TODO: Needs a workunit, when we have a context here. return True except (Bootstrapper.Error, Ivy.Error) as e: print('ANTLR generation failed! %s' % e, file=sys.stderr) return False
def run_antlrs(self, output_dir): # TODO(John Sirois): graduate to a JvmToolTask and/or merge with the java code gen AntlrGen # task. args = [ '-dependency', 'org.antlr', 'antlr', self.target.antlr_version, '-types', 'jar', '-main', 'org.antlr.Tool', '--', '-fo', output_dir ] for source in self.target.sources_relative_to_buildroot(): abs_path = os.path.join(get_buildroot(), source) args.append(abs_path) try: ivy = Bootstrapper.default_ivy() ivy.execute(args=args) # TODO: Needs a workunit, when we have a context here. except (Bootstrapper.Error, Ivy.Error) as e: raise TaskError('ANTLR generation failed! {0}'.format(e))
def run_antlrs(self, output_dir): # TODO(John Sirois): graduate to a JvmToolTask and/or merge with the java code gen AntlrGen # task. args = [ '-dependency', 'org.antlr', 'antlr', self.target.antlr_version, '-types', 'jar', '-main', 'org.antlr.Tool', '--', '-fo', output_dir ] for source in self.target.sources_relative_to_buildroot(): abs_path = os.path.join(get_buildroot(), source) args.append(abs_path) try: ivy = Bootstrapper.default_ivy() ivy.execute( args=args ) # TODO: Needs a workunit, when we have a context here. except (Bootstrapper.Error, Ivy.Error) as e: raise TaskError('ANTLR generation failed! {0}'.format(e))
def _exec_ivy( self, target_workdir, targets, args, executor=None, confs=None, ivy=None, workunit_name="ivy", use_soft_excludes=False, resolve_hash_name=None, ): ivy_jvm_options = self.get_options().jvm_options[:] # Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng. ivy_jvm_options.append("-Dsun.io.useCanonCaches=false") ivy = ivy or Bootstrapper.default_ivy() ivyxml = os.path.join(target_workdir, "ivy.xml") ivy_args = ["-ivy", ivyxml] confs_to_resolve = confs or ("default",) ivy_args.append("-confs") ivy_args.extend(confs_to_resolve) ivy_args.extend(args) # TODO(John Sirois): merge the code below into IvyUtils or up here; either way, better # diagnostics can be had in `IvyUtils.generate_ivy` if this is done. # See: https://github.com/pantsbuild/pants/issues/2239 try: jars, excludes = IvyUtils.calculate_classpath(targets, gather_excludes=not use_soft_excludes) with IvyUtils.ivy_lock: IvyUtils.generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve, resolve_hash_name) runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor) try: result = execute_runner( runner, workunit_factory=self.context.new_workunit, workunit_name=workunit_name ) if result != 0: raise self.Error("Ivy returned {result}. cmd={cmd}".format(result=result, cmd=runner.cmd)) except runner.executor.Error as e: raise self.Error(e) except IvyUtils.IvyError as e: raise self.Error("Failed to prepare ivy resolve: {}".format(e))
def exec_ivy(self, target_workdir, targets, args, executor=None, confs=None, ivy=None, workunit_name='ivy', jars=None, use_soft_excludes=False, resolve_hash_name=None): ivy_jvm_options = copy.copy(self.get_options().jvm_options) # Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng. ivy_jvm_options.append('-Dsun.io.useCanonCaches=false') ivy = ivy or Bootstrapper.default_ivy() ivyxml = os.path.join(target_workdir, 'ivy.xml') if not jars: automatic_excludes = self.get_options().automatic_excludes jars, excludes = IvyUtils.calculate_classpath(targets, gather_excludes=not use_soft_excludes, automatic_excludes=automatic_excludes) else: excludes = set() ivy_args = ['-ivy', ivyxml] confs_to_resolve = confs or ['default'] ivy_args.append('-confs') ivy_args.extend(confs_to_resolve) ivy_args.extend(args) with IvyUtils.ivy_lock: IvyUtils.generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve, resolve_hash_name) runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor) try: result = execute_runner(runner, workunit_factory=self.context.new_workunit, workunit_name=workunit_name) if result != 0: raise TaskError('Ivy returned {result}. cmd={cmd}'.format(result=result, cmd=runner.cmd)) except runner.executor.Error as e: raise TaskError(e)
def mapjars(self, genmap, target, executor, jars=None): """Resolves jars for the target and stores their locations in genmap. :param genmap: The jar_dependencies ProductMapping entry for the required products. :param target: The target whose jar dependencies are being retrieved. :param jars: If specified, resolves the given jars rather than :type jars: List of :class:`pants.backend.jvm.targets.jar_dependency.JarDependency` (jar()) objects. """ mapdir = os.path.join(self.workdir, 'mapped-jars', target.id) safe_mkdir(mapdir, clean=True) ivyargs = [ '-retrieve', '%s/[organisation]/[artifact]/[conf]/' '[organisation]-[artifact]-[revision](-[classifier]).[ext]' % mapdir, '-symlink', ] confs = maybe_list(target.payload.get_field_value('configurations') or []) self.exec_ivy(mapdir, [target], executor=executor, args=ivyargs, confs=confs, ivy=Bootstrapper.default_ivy(), workunit_name='map-jars', jars=jars) for org in os.listdir(mapdir): orgdir = os.path.join(mapdir, org) if os.path.isdir(orgdir): for name in os.listdir(orgdir): artifactdir = os.path.join(orgdir, name) if os.path.isdir(artifactdir): for conf in os.listdir(artifactdir): confdir = os.path.join(artifactdir, conf) for f in os.listdir(confdir): # TODO(John Sirois): kill the org and (org, name) exclude mappings in favor of a # conf whitelist genmap.add(org, confdir).append(f) genmap.add((org, name), confdir).append(f) genmap.add(target, confdir).append(f) genmap.add((target, conf), confdir).append(f) genmap.add((org, name, conf), confdir).append(f)
def do_resolve(cls, executor, extra_args, ivyxml, jvm_options, workdir_report_paths_by_conf, confs, ivy_cache_dir, ivy_cache_classpath_filename, resolve_hash_name, workunit_factory, workunit_name): """Execute Ivy with the given ivy.xml and copies all relevant files into the workdir. This method does an Ivy resolve, which may be either a Pants resolve or a Pants fetch depending on whether there is an existing frozen resolution. After it is run, the Ivy reports are copied into the workdir at the paths specified by workdir_report_paths_by_conf along with a file containing a list of all the requested artifacts and their transitive dependencies. :param executor: A JVM executor to use to invoke ivy. :param extra_args: Extra arguments to pass to ivy. :param ivyxml: The input ivy.xml containing the dependencies to resolve. :param jvm_options: A list of jvm option strings to use for the ivy invoke, or None. :param workdir_report_paths_by_conf: A dict mapping confs to report paths in the workdir. :param confs: The confs used in the resolve. :param resolve_hash_name: The hash to use as the module name for finding the ivy report file. :param workunit_factory: A workunit factory for the ivy invoke, or None. :param workunit_name: A workunit name for the ivy invoke, or None. """ ivy = Bootstrapper.default_ivy(bootstrap_workunit_factory=workunit_factory) with safe_concurrent_creation(ivy_cache_classpath_filename) as raw_target_classpath_file_tmp: extra_args = extra_args or [] args = ['-cachepath', raw_target_classpath_file_tmp] + extra_args with cls._ivy_lock: cls._exec_ivy(ivy, confs, ivyxml, args, jvm_options=jvm_options, executor=executor, workunit_name=workunit_name, workunit_factory=workunit_factory) if not os.path.exists(raw_target_classpath_file_tmp): raise cls.IvyError('Ivy failed to create classpath file at {}' .format(raw_target_classpath_file_tmp)) cls._copy_ivy_reports(workdir_report_paths_by_conf, confs, ivy_cache_dir, resolve_hash_name) logger.debug('Moved ivy classfile file to {dest}' .format(dest=ivy_cache_classpath_filename))
def mapjars(self, genmap, target, executor, workunit_factory=None): """ Parameters: genmap: the jar_dependencies ProductMapping entry for the required products. target: the target whose jar dependencies are being retrieved. """ mapdir = os.path.join(self.mapto_dir(), target.id) safe_mkdir(mapdir, clean=True) ivyargs = [ '-retrieve', '%s/[organisation]/[artifact]/[conf]/' '[organisation]-[artifact]-[revision](-[classifier]).[ext]' % mapdir, '-symlink', ] self.exec_ivy(mapdir, [target], ivyargs, confs=target.payload.configurations, ivy=Bootstrapper.default_ivy(executor), workunit_factory=workunit_factory, workunit_name='map-jars') for org in os.listdir(mapdir): orgdir = os.path.join(mapdir, org) if os.path.isdir(orgdir): for name in os.listdir(orgdir): artifactdir = os.path.join(orgdir, name) if os.path.isdir(artifactdir): for conf in os.listdir(artifactdir): confdir = os.path.join(artifactdir, conf) for f in os.listdir(confdir): if self.is_mappable_artifact(org, name, f): # TODO(John Sirois): kill the org and (org, name) exclude mappings in favor of a # conf whitelist genmap.add(org, confdir).append(f) genmap.add((org, name), confdir).append(f) genmap.add(target, confdir).append(f) genmap.add((target, conf), confdir).append(f) genmap.add((org, name, conf), confdir).append(f)
def publish(self, ivyxml_path, jar, entry, repo, published): """Run ivy to publish a jar. ivyxml_path is the path to the ivy file; published is a list of jars published so far (including this one). entry is a pushdb entry.""" jvm_options = self._ivy_jvm_options(repo) resolver = repo['resolver'] path = repo.get('path') try: ivy = Bootstrapper.default_ivy() except Bootstrapper.Error as e: raise TaskError('Failed to push {0}! {1}'.format(pushdb_coordinate(jar, entry), e)) ivysettings = self.generate_ivysettings(ivy, published, publish_local=path) args = [ '-settings', ivysettings, '-ivy', ivyxml_path, '-deliverto', '%s/[organisation]/[module]/ivy-[revision].xml' % self.workdir, '-publish', resolver, '-publishpattern', '%s/[organisation]/[module]/' '[artifact]-[revision](-[classifier]).[ext]' % self.workdir, '-revision', entry.version().version(), '-m2compatible', ] # TODO(John Sirois): global logging options should be hidden behind some sort of log manager # that we can: # a.) obtain a handle to (dependency injection or manual plumbing) # b.) query for log detail, ie: `if log_manager.is_verbose:` if self.get_options().level == 'debug': args.append('-verbose') if self.local_snapshot: args.append('-overwrite') try: ivy.execute(jvm_options=jvm_options, args=args, workunit_factory=self.context.new_workunit, workunit_name='jar-publish') except Ivy.Error as e: raise TaskError('Failed to push {0}! {1}'.format(pushdb_coordinate(jar, entry), e))
def _do_resolve(self, confs, executor, extra_args, global_vts, pinned_artifacts, raw_target_classpath_file, resolve_hash_name, resolve_workdir, workunit_name): safe_mkdir(resolve_workdir) ivy = Bootstrapper.default_ivy(bootstrap_workunit_factory=self.context.new_workunit) with safe_concurrent_creation(raw_target_classpath_file) as raw_target_classpath_file_tmp: args = ['-cachepath', raw_target_classpath_file_tmp] + extra_args targets = global_vts.targets # TODO(John Sirois): merge the code below into IvyUtils or up here; either way, better # diagnostics can be had in `IvyUtils.generate_ivy` if this is done. # See: https://github.com/pantsbuild/pants/issues/2239 jars, global_excludes = IvyUtils.calculate_classpath(targets) # Don't pass global excludes to ivy when using soft excludes. if self.get_options().soft_excludes: global_excludes = [] ivyxml = self._ivy_xml_path(resolve_workdir) with IvyUtils.ivy_lock: try: IvyUtils.generate_ivy(targets, jars, global_excludes, ivyxml, confs, resolve_hash_name, pinned_artifacts) except IvyUtils.IvyError as e: raise self.Error('Failed to prepare ivy resolve: {}'.format(e)) self._exec_ivy(ivy, executor, confs, ivyxml, args, workunit_name) # Copy ivy resolve file into resolve workdir. for conf in confs: atomic_copy(IvyUtils.xml_report_path(self.ivy_cache_dir, resolve_hash_name, conf), self._resolve_report_path(resolve_workdir, conf)) if not os.path.exists(raw_target_classpath_file_tmp): raise self.Error('Ivy failed to create classpath file at {}' .format(raw_target_classpath_file_tmp)) logger.debug('Moved ivy classfile file to {dest}'.format(dest=raw_target_classpath_file))
def mapjars(self, genmap, target, executor, workunit_factory=None): """ Parameters: genmap: the jar_dependencies ProductMapping entry for the required products. target: the target whose jar dependencies are being retrieved. """ mapdir = os.path.join(self.mapto_dir(), target.id) safe_mkdir(mapdir, clean=True) ivyargs = [ '-retrieve', '%s/[organisation]/[artifact]/[conf]/' '[organisation]-[artifact]-[revision](-[classifier]).[ext]' % mapdir, '-symlink', ] self.exec_ivy(mapdir, [target], ivyargs, confs=target.payload.configurations, ivy=Bootstrapper.default_ivy(executor), workunit_factory=workunit_factory, workunit_name='map-jars') for org in os.listdir(mapdir): orgdir = os.path.join(mapdir, org) if os.path.isdir(orgdir): for name in os.listdir(orgdir): artifactdir = os.path.join(orgdir, name) if os.path.isdir(artifactdir): for conf in os.listdir(artifactdir): confdir = os.path.join(artifactdir, conf) for f in os.listdir(confdir): if self.is_mappable_artifact(org, name, f): # TODO(John Sirois): kill the org and (org, name) exclude mappings in favor of a # conf whitelist genmap.add(org, confdir).append(f) genmap.add((org, name), confdir).append(f) genmap.add(target, confdir).append(f) genmap.add((target, conf), confdir).append(f) genmap.add((org, name, conf), confdir).append(f)
def test_default_ivy(self): with subsystem_instance(IvySubsystem): ivy = Bootstrapper.default_ivy() self.assertIsNotNone(ivy.ivy_cache_dir) self.assertIsNone(ivy.ivy_settings)
def mapjars(self, genmap, target, executor, jars=None): """Resolves jars for the target and stores their locations in genmap. :param genmap: The jar_dependencies ProductMapping entry for the required products. :param target: The target whose jar dependencies are being retrieved. :param jars: If specified, resolves the given jars rather than :type jars: List of :class:`pants.backend.jvm.targets.jar_dependency.JarDependency` (jar()) objects. Here is an example of what the resulting genmap looks like after sucessfully mapping a JarLibrary target with a single JarDependency: ProductMapping(ivy_imports) { # target UnpackedJars(BuildFileAddress(.../unpack/BUILD, foo)) => .../.pants.d/test/IvyImports/mapped-jars/unpack.foo/com.example/bar/default [u'com.example-bar-0.0.1.jar'] # (org, name) (u'com.example', u'bar') => .../.pants.d/test/IvyImports/mapped-jars/unpack.foo/com.example/bar/default [u'com.example-bar-0.0.1.jar'] # (org) com.example => .../.pants.d/test/IvyImports/mapped-jars/unpack.foo/com.example/bar/default [u'com.example-bar-0.0.1.jar'] # (target) (UnpackedJars(BuildFileAddress(.../unpack/BUILD, foo)), u'default') => .../.pants.d/test/IvyImports/mapped-jars/unpack.foo/com.example/bar/default [u'com.example-bar-0.0.1.jar'] # (org, name, conf) (u'com.example', u'bar', u'default') => .../.pants.d/test/IvyImports/mapped-jars/unpack.foo/com.example/bar/default [u'com.example-bar-0.0.1.jar'] """ mapdir = self.mapjar_workdir(target) safe_mkdir(mapdir, clean=True) ivyargs = [ '-retrieve', '{}/[organisation]/[artifact]/[conf]/' '[organisation]-[artifact]-[revision](-[classifier]).[ext]'.format(mapdir), '-symlink', ] confs = maybe_list(target.payload.get_field_value('configurations') or []) self.exec_ivy(mapdir, [target], executor=executor, args=ivyargs, confs=confs, ivy=Bootstrapper.default_ivy(), workunit_name='map-jars', jars=jars, use_soft_excludes=False) for org in os.listdir(mapdir): orgdir = os.path.join(mapdir, org) if os.path.isdir(orgdir): for name in os.listdir(orgdir): artifactdir = os.path.join(orgdir, name) if os.path.isdir(artifactdir): for conf in os.listdir(artifactdir): confdir = os.path.join(artifactdir, conf) for f in os.listdir(confdir): # TODO(John Sirois): kill the org and (org, name) exclude mappings in favor of a # conf whitelist genmap.add(org, confdir).append(f) genmap.add((org, name), confdir).append(f) genmap.add(target, confdir).append(f) genmap.add((target, conf), confdir).append(f) genmap.add((org, name, conf), confdir).append(f)
def ivy_resolve(self, targets, executor=None, silent=False, workunit_name=None, confs=None, custom_args=None): """Executes an ivy resolve for the relevant subset of the given targets. Returns the resulting classpath, and the set of relevant targets. Also populates the 'ivy_resolve_symlink_map' product for jars resulting from the resolve.""" if not targets: return ([], set()) # NOTE: Always pass all the targets to exec_ivy, as they're used to calculate the name of # the generated module, which in turn determines the location of the XML report file # ivy generates. We recompute this name from targets later in order to find that file. # TODO: This is fragile. Refactor so that we're not computing the name twice. ivy = Bootstrapper.default_ivy(bootstrap_workunit_factory=self.context.new_workunit) ivy_workdir = os.path.join(self.context.options.for_global_scope().pants_workdir, 'ivy') fingerprint_strategy = IvyResolveFingerprintStrategy(confs) with self.invalidated(targets, invalidate_dependents=False, silent=silent, fingerprint_strategy=fingerprint_strategy) as invalidation_check: if not invalidation_check.all_vts: return ([], set()) global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts) # If a report file is not present, we need to exec ivy, even if all the individual # targets up to date... See https://rbcommons.com/s/twitter/r/2015 report_missing = False report_confs = confs or ['default'] report_paths = [] for conf in report_confs: report_path = IvyUtils.xml_report_path(global_vts.targets, conf) if not os.path.exists(report_path): report_missing = True break else: report_paths.append(report_path) target_workdir = os.path.join(ivy_workdir, global_vts.cache_key.hash) target_classpath_file = os.path.join(target_workdir, 'classpath') raw_target_classpath_file = target_classpath_file + '.raw' raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp' # A common dir for symlinks into the ivy2 cache. This ensures that paths to jars # in artifact-cached analysis files are consistent across systems. # Note that we have one global, well-known symlink dir, again so that paths are # consistent across builds. symlink_dir = os.path.join(ivy_workdir, 'jars') # Note that it's possible for all targets to be valid but for no classpath file to exist at # target_classpath_file, e.g., if we previously built a superset of targets. if report_missing or invalidation_check.invalid_vts or not os.path.exists(raw_target_classpath_file): args = ['-cachepath', raw_target_classpath_file_tmp] + (custom_args if custom_args else []) self.exec_ivy( target_workdir=target_workdir, targets=global_vts.targets, args=args, executor=executor, ivy=ivy, workunit_name=workunit_name, confs=confs, use_soft_excludes=self.get_options().soft_excludes) if not os.path.exists(raw_target_classpath_file_tmp): raise TaskError('Ivy failed to create classpath file at {}' .format(raw_target_classpath_file_tmp)) shutil.move(raw_target_classpath_file_tmp, raw_target_classpath_file) logger.debug('Moved ivy classfile file to {dest}'.format(dest=raw_target_classpath_file)) if self.artifact_cache_writes_enabled(): self.update_artifact_cache([(global_vts, [raw_target_classpath_file])]) else: logger.debug("Using previously resolved reports: {}".format(report_paths)) # Make our actual classpath be symlinks, so that the paths are uniform across systems. # Note that we must do this even if we read the raw_target_classpath_file from the artifact # cache. If we cache the target_classpath_file we won't know how to create the symlinks. with IvyTaskMixin.symlink_map_lock: products = self.context.products existing_symlinks_map = products.get_data('ivy_resolve_symlink_map', lambda: dict()) symlink_map = IvyUtils.symlink_cachepath( IvySubsystem.global_instance().get_options().cache_dir, raw_target_classpath_file, symlink_dir, target_classpath_file, existing_symlinks_map) existing_symlinks_map.update(symlink_map) with IvyUtils.cachepath(target_classpath_file) as classpath: stripped_classpath = [path.strip() for path in classpath] return (stripped_classpath, global_vts.targets)
def ivy_resolve(self, targets, executor=None, silent=False, workunit_name=None, confs=None, custom_args=None): """Executes an ivy resolve for the relevant subset of the given targets. :returns: the resulting classpath, and the unique part of the name used for the resolution report (a hash). Also populates the 'ivy_resolve_symlink_map' product for jars resulting from the resolve.""" if not targets: return ([], None) ivy = Bootstrapper.default_ivy(bootstrap_workunit_factory=self.context.new_workunit) ivy_workdir = os.path.join(self.context.options.for_global_scope().pants_workdir, 'ivy') fingerprint_strategy = IvyResolveFingerprintStrategy(confs) with self.invalidated(targets, invalidate_dependents=False, silent=silent, fingerprint_strategy=fingerprint_strategy) as invalidation_check: if not invalidation_check.all_vts: return ([], None) global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts) # If a report file is not present, we need to exec ivy, even if all the individual # targets up to date... See https://rbcommons.com/s/twitter/r/2015 report_missing = False report_confs = confs or ['default'] report_paths = [] resolve_hash_name = global_vts.cache_key.hash for conf in report_confs: report_path = IvyUtils.xml_report_path(resolve_hash_name, conf) if not os.path.exists(report_path): report_missing = True break else: report_paths.append(report_path) target_workdir = os.path.join(ivy_workdir, resolve_hash_name) target_classpath_file = os.path.join(target_workdir, 'classpath') raw_target_classpath_file = target_classpath_file + '.raw' raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp' # A common dir for symlinks into the ivy2 cache. This ensures that paths to jars # in artifact-cached analysis files are consistent across systems. # Note that we have one global, well-known symlink dir, again so that paths are # consistent across builds. symlink_dir = os.path.join(ivy_workdir, 'jars') # Note that it's possible for all targets to be valid but for no classpath file to exist at # target_classpath_file, e.g., if we previously built a superset of targets. if report_missing or invalidation_check.invalid_vts or not os.path.exists(raw_target_classpath_file): args = ['-cachepath', raw_target_classpath_file_tmp] + (custom_args if custom_args else []) self.exec_ivy( target_workdir=target_workdir, targets=global_vts.targets, args=args, executor=executor, ivy=ivy, workunit_name=workunit_name, confs=confs, use_soft_excludes=self.get_options().soft_excludes, resolve_hash_name=resolve_hash_name) if not os.path.exists(raw_target_classpath_file_tmp): raise TaskError('Ivy failed to create classpath file at {}' .format(raw_target_classpath_file_tmp)) shutil.move(raw_target_classpath_file_tmp, raw_target_classpath_file) logger.debug('Moved ivy classfile file to {dest}'.format(dest=raw_target_classpath_file)) if self.artifact_cache_writes_enabled(): self.update_artifact_cache([(global_vts, [raw_target_classpath_file])]) else: logger.debug("Using previously resolved reports: {}".format(report_paths)) # Make our actual classpath be symlinks, so that the paths are uniform across systems. # Note that we must do this even if we read the raw_target_classpath_file from the artifact # cache. If we cache the target_classpath_file we won't know how to create the symlinks. with IvyTaskMixin.symlink_map_lock: products = self.context.products existing_symlinks_map = products.get_data('ivy_resolve_symlink_map', lambda: dict()) symlink_map = IvyUtils.symlink_cachepath( IvySubsystem.global_instance().get_options().cache_dir, raw_target_classpath_file, symlink_dir, target_classpath_file, existing_symlinks_map) existing_symlinks_map.update(symlink_map) with IvyUtils.cachepath(target_classpath_file) as classpath: stripped_classpath = [path.strip() for path in classpath] return (stripped_classpath, resolve_hash_name)
def _ivy_resolve(self, targets, executor=None, silent=False, workunit_name=None, confs=None, extra_args=None, invalidate_dependents=False, pinned_artifacts=None): """Resolves external dependencies for the given targets. If there are no targets suitable for jvm transitive dependency resolution, an empty result is returned, ie: ([], {}, None). :param targets: The targets to resolve jvm dependencies for. :type targets: :class:`collections.Iterable` of :class:`pants.build_graph.target.Target` :param executor: A java executor to run ivy with. :type executor: :class:`pants.java.executor.Executor` :param confs: The ivy configurations to resolve; ('default',) by default. :type confs: :class:`collections.Iterable` of string :param extra_args: Any extra command line arguments to pass to ivy. :type extra_args: list of string :param bool invalidate_dependents: `True` to invalidate dependents of targets that needed to be resolved. :returns: A tuple of the classpath, a mapping from ivy cache jars to their linked location under .pants.d, and the id of the reports associated with the resolve. :rtype: tuple of (list, dict, string) """ if not targets: return [], {}, None confs = confs or ('default',) extra_args = extra_args or [] fingerprint_strategy = IvyResolveFingerprintStrategy(confs) # NB: See class pydoc regarding `use_cache=False`. with self.invalidated(targets, invalidate_dependents=invalidate_dependents, silent=silent, fingerprint_strategy=fingerprint_strategy, use_cache=False) as invalidation_check: if not invalidation_check.all_vts: return [], {}, None global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts) resolve_hash_name = global_vts.cache_key.hash ivy_workdir = os.path.join(self.context.options.for_global_scope().pants_workdir, 'ivy') target_workdir = os.path.join(ivy_workdir, resolve_hash_name) target_classpath_file = os.path.join(target_workdir, 'classpath') raw_target_classpath_file = target_classpath_file + '.raw' # If a report file is not present, we need to exec ivy, even if all the individual # targets are up to date. See https://rbcommons.com/s/twitter/r/2015. # Note that it's possible for all targets to be valid but for no classpath file to exist at # target_classpath_file, e.g., if we previously built a superset of targets. any_report_missing, existing_report_paths = self._collect_existing_reports(confs, resolve_hash_name) if (invalidation_check.invalid_vts or any_report_missing or not os.path.exists(raw_target_classpath_file)): ivy = Bootstrapper.default_ivy(bootstrap_workunit_factory=self.context.new_workunit) raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp' args = ['-cachepath', raw_target_classpath_file_tmp] + extra_args self._exec_ivy( target_workdir=target_workdir, targets=global_vts.targets, args=args, executor=executor, ivy=ivy, workunit_name=workunit_name, confs=confs, use_soft_excludes=self.get_options().soft_excludes, resolve_hash_name=resolve_hash_name, pinned_artifacts=pinned_artifacts) if not os.path.exists(raw_target_classpath_file_tmp): raise self.Error('Ivy failed to create classpath file at {}' .format(raw_target_classpath_file_tmp)) shutil.move(raw_target_classpath_file_tmp, raw_target_classpath_file) logger.debug('Moved ivy classfile file to {dest}'.format(dest=raw_target_classpath_file)) else: logger.debug("Using previously resolved reports: {}".format(existing_report_paths)) # Make our actual classpath be symlinks, so that the paths are uniform across systems. # Note that we must do this even if we read the raw_target_classpath_file from the artifact # cache. If we cache the target_classpath_file we won't know how to create the symlinks. with IvyTaskMixin.symlink_map_lock: # A common dir for symlinks into the ivy2 cache. This ensures that paths to jars # in artifact-cached analysis files are consistent across systems. # Note that we have one global, well-known symlink dir, again so that paths are # consistent across builds. symlink_dir = os.path.join(ivy_workdir, 'jars') symlink_map = IvyUtils.symlink_cachepath(self.ivy_cache_dir, raw_target_classpath_file, symlink_dir, target_classpath_file) classpath = IvyUtils.load_classpath_from_cachepath(target_classpath_file) return classpath, symlink_map, resolve_hash_name
def mapjars(self, genmap, target, executor, jars=None): """Resolves jars for the target and stores their locations in genmap. :param genmap: The jar_dependencies ProductMapping entry for the required products. :param target: The target whose jar dependencies are being retrieved. :param jars: If specified, resolves the given jars rather than :type jars: List of :class:`pants.backend.jvm.targets.jar_dependency.JarDependency` (jar()) objects. Here is an example of what the resulting genmap looks like after sucessfully mapping a JarLibrary target with a single JarDependency: ProductMapping(ivy_imports) { # target UnpackedJars(BuildFileAddress(.../unpack/BUILD, foo)) => .../.pants.d/test/IvyImports/mapped-jars/unpack.foo/com.example/bar/default [u'com.example-bar-0.0.1.jar'] # (org, name) (u'com.example', u'bar') => .../.pants.d/test/IvyImports/mapped-jars/unpack.foo/com.example/bar/default [u'com.example-bar-0.0.1.jar'] # (org) com.example => .../.pants.d/test/IvyImports/mapped-jars/unpack.foo/com.example/bar/default [u'com.example-bar-0.0.1.jar'] # (target) (UnpackedJars(BuildFileAddress(.../unpack/BUILD, foo)), u'default') => .../.pants.d/test/IvyImports/mapped-jars/unpack.foo/com.example/bar/default [u'com.example-bar-0.0.1.jar'] # (org, name, conf) (u'com.example', u'bar', u'default') => .../.pants.d/test/IvyImports/mapped-jars/unpack.foo/com.example/bar/default [u'com.example-bar-0.0.1.jar'] """ mapdir = self.mapjar_workdir(target) safe_mkdir(mapdir, clean=True) ivyargs = self._get_ivy_args(mapdir) confs = maybe_list(target.payload.get_field_value('configurations') or []) self.exec_ivy(mapdir, [target], executor=executor, args=ivyargs, confs=confs, ivy=Bootstrapper.default_ivy(), workunit_name='map-jars', jars=jars, use_soft_excludes=False) for org in os.listdir(mapdir): orgdir = os.path.join(mapdir, org) if os.path.isdir(orgdir): for name in os.listdir(orgdir): artifactdir = os.path.join(orgdir, name) if os.path.isdir(artifactdir): for conf in os.listdir(artifactdir): confdir = os.path.join(artifactdir, conf) for f in os.listdir(confdir): # TODO(John Sirois): kill the org and (org, name) exclude mappings in favor of a # conf whitelist genmap.add(org, confdir).append(f) genmap.add((org, name), confdir).append(f) genmap.add(target, confdir).append(f) genmap.add((target, conf), confdir).append(f) genmap.add((org, name, conf), confdir).append(f)
def publish(self, publications, jar, version, repo, published): """Run ivy to publish a jar. :param str ivyxml_path: The path to the ivy file. :param list published: A list of jars published so far (including this one). """ try: ivy = Bootstrapper.default_ivy() except Bootstrapper.Error as e: raise TaskError('Failed to push {0}! {1}'.format(jar, e)) if ivy.ivy_settings is None: raise TaskError( 'An ivysettings.xml with writeable resolvers is required for publishing, ' 'but none was configured.') path = repo.get('path') ivysettings = self.generate_ivysettings(self.fetch_ivysettings(ivy), published, publish_local=path) ivyxml = self.generate_ivy(jar, version, publications) resolver = repo['resolver'] args = [ '-settings', ivysettings, '-ivy', ivyxml, # Without this setting, the ivy.xml is delivered to the CWD, littering the workspace. We # don't need the ivy.xml, so just give it path under the workdir we won't use. '-deliverto', ivyxml + '.unused', '-publish', resolver, '-publishpattern', '{}/[organisation]/[module]/' '[artifact]-[revision](-[classifier]).[ext]'.format(self.workdir), '-revision', version, '-m2compatible', ] if self.get_options().level == 'debug': args.append('-verbose') if self.get_options().local: args.append('-overwrite') if not self.get_options().dryrun: try: ivy.execute(jvm_options=self._ivy_jvm_options(repo), args=args, workunit_factory=self.context.new_workunit, workunit_name='ivy-publish') except Ivy.Error as e: raise TaskError('Failed to push {0}! {1}'.format(jar, e)) else: print("\nDRYRUN- would have pushed:{} using {} resolver.".format( self.jar_coordinate(jar, version), resolver))
def test_default_ivy(self): ivy = Bootstrapper.default_ivy() self.assertIsNotNone(ivy.ivy_cache_dir) self.assertIsNotNone(ivy.ivy_settings)
def ivy_resolve(self, targets, executor=None, symlink_ivyxml=False, silent=False, workunit_name=None, workunit_labels=None): # NOTE: Always pass all the targets to exec_ivy, as they're used to calculate the name of # the generated module, which in turn determines the location of the XML report file # ivy generates. We recompute this name from targets later in order to find that file. # TODO: This is fragile. Refactor so that we're not computing the name twice. if executor and not isinstance(executor, Executor): raise ValueError('The executor must be an Executor instance, given %s of type %s' % (executor, type(executor))) ivy = Bootstrapper.default_ivy(java_executor=executor, bootstrap_workunit_factory=self.context.new_workunit) if not targets: return [] ivy_workdir = os.path.join(self.context.config.getdefault('pants_workdir'), 'ivy') ivy_utils = IvyUtils(config=self.context.config, options=self.context.options, log=self.context.log) fingerprint_strategy = IvyResolveFingerprintStrategy() with self.invalidated(targets, invalidate_dependents=True, silent=silent, fingerprint_strategy=fingerprint_strategy) as invalidation_check: global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts) target_workdir = os.path.join(ivy_workdir, global_vts.cache_key.hash) target_classpath_file = os.path.join(target_workdir, 'classpath') raw_target_classpath_file = target_classpath_file + '.raw' raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp' # A common dir for symlinks into the ivy2 cache. This ensures that paths to jars # in artifact-cached analysis files are consistent across systems. # Note that we have one global, well-known symlink dir, again so that paths are # consistent across builds. symlink_dir = os.path.join(ivy_workdir, 'jars') # Note that it's possible for all targets to be valid but for no classpath file to exist at # target_classpath_file, e.g., if we previously built a superset of targets. if invalidation_check.invalid_vts or not os.path.exists(raw_target_classpath_file): args = ['-cachepath', raw_target_classpath_file_tmp] def exec_ivy(): ivy_utils.exec_ivy( target_workdir=target_workdir, targets=targets, args=args, ivy=ivy, workunit_name='ivy', workunit_factory=self.context.new_workunit, symlink_ivyxml=symlink_ivyxml) if workunit_name: with self.context.new_workunit(name=workunit_name, labels=workunit_labels or []): exec_ivy() else: exec_ivy() if not os.path.exists(raw_target_classpath_file_tmp): raise TaskError('Ivy failed to create classpath file at %s' % raw_target_classpath_file_tmp) shutil.move(raw_target_classpath_file_tmp, raw_target_classpath_file) if self.artifact_cache_writes_enabled(): self.update_artifact_cache([(global_vts, [raw_target_classpath_file])]) # Make our actual classpath be symlinks, so that the paths are uniform across systems. # Note that we must do this even if we read the raw_target_classpath_file from the artifact # cache. If we cache the target_classpath_file we won't know how to create the symlinks. symlink_map = IvyUtils.symlink_cachepath(self.context.ivy_home, raw_target_classpath_file, symlink_dir, target_classpath_file) with IvyTaskMixin.symlink_map_lock: all_symlinks_map = self.context.products.get_data('symlink_map') or defaultdict(list) for path, symlink in symlink_map.items(): all_symlinks_map[os.path.realpath(path)].append(symlink) self.context.products.safe_create_data('symlink_map', lambda: all_symlinks_map) with IvyUtils.cachepath(target_classpath_file) as classpath: stripped_classpath = [path.strip() for path in classpath] return [path for path in stripped_classpath if ivy_utils.is_classpath_artifact(path)]
def ivy_resolve(self, targets, executor=None, silent=False, workunit_name=None, confs=None, custom_args=None): """Executes an ivy resolve for the relevant subset of the given targets. Returns the resulting classpath, and the set of relevant targets. Also populates the 'ivy_resolve_symlink_map' product for jars resulting from the resolve.""" if not targets: return ([], set()) # NOTE: Always pass all the targets to exec_ivy, as they're used to calculate the name of # the generated module, which in turn determines the location of the XML report file # ivy generates. We recompute this name from targets later in order to find that file. # TODO: This is fragile. Refactor so that we're not computing the name twice. ivy = Bootstrapper.default_ivy( bootstrap_workunit_factory=self.context.new_workunit) ivy_workdir = os.path.join( self.context.options.for_global_scope().pants_workdir, 'ivy') fingerprint_strategy = IvyResolveFingerprintStrategy() with self.invalidated(targets, invalidate_dependents=False, silent=silent, fingerprint_strategy=fingerprint_strategy ) as invalidation_check: if not invalidation_check.all_vts: return ([], set()) global_vts = VersionedTargetSet.from_versioned_targets( invalidation_check.all_vts) target_workdir = os.path.join(ivy_workdir, global_vts.cache_key.hash) target_classpath_file = os.path.join(target_workdir, 'classpath') raw_target_classpath_file = target_classpath_file + '.raw' raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp' # A common dir for symlinks into the ivy2 cache. This ensures that paths to jars # in artifact-cached analysis files are consistent across systems. # Note that we have one global, well-known symlink dir, again so that paths are # consistent across builds. symlink_dir = os.path.join(ivy_workdir, 'jars') # Note that it's possible for all targets to be valid but for no classpath file to exist at # target_classpath_file, e.g., if we previously built a superset of targets. if invalidation_check.invalid_vts or not os.path.exists( raw_target_classpath_file): args = ['-cachepath', raw_target_classpath_file_tmp ] + (custom_args if custom_args else []) self.exec_ivy(target_workdir=target_workdir, targets=global_vts.targets, args=args, executor=executor, ivy=ivy, workunit_name=workunit_name, confs=confs) if not os.path.exists(raw_target_classpath_file_tmp): raise TaskError( 'Ivy failed to create classpath file at %s' % raw_target_classpath_file_tmp) shutil.move(raw_target_classpath_file_tmp, raw_target_classpath_file) logger.debug('Moved ivy classfile file to {dest}'.format( dest=raw_target_classpath_file)) if self.artifact_cache_writes_enabled(): self.update_artifact_cache([(global_vts, [raw_target_classpath_file])]) # Make our actual classpath be symlinks, so that the paths are uniform across systems. # Note that we must do this even if we read the raw_target_classpath_file from the artifact # cache. If we cache the target_classpath_file we won't know how to create the symlinks. with IvyTaskMixin.symlink_map_lock: products = self.context.products existing_symlinks_map = products.get_data( 'ivy_resolve_symlink_map', lambda: dict()) symlink_map = IvyUtils.symlink_cachepath( ivy.ivy_cache_dir, raw_target_classpath_file, symlink_dir, target_classpath_file, existing_symlinks_map) existing_symlinks_map.update(symlink_map) with IvyUtils.cachepath(target_classpath_file) as classpath: stripped_classpath = [path.strip() for path in classpath] return (stripped_classpath, global_vts.targets)
def publish(self, publications, jar, entry, repo, published): """Run ivy to publish a jar. ivyxml_path is the path to the ivy file; published is a list of jars published so far (including this one). entry is a pushdb entry. """ try: ivy = Bootstrapper.default_ivy() except Bootstrapper.Error as e: raise TaskError( f"Failed to push {pushdb_coordinate(jar, entry)}! {e!r}") path = repo.get("path") ivysettings = self.generate_ivysettings(ivy, published, publish_local=path) version = entry.version().version() ivyxml = self.generate_ivy(jar, version, publications) resolver = repo["resolver"] args = [ "-settings", ivysettings, "-ivy", ivyxml, # Without this setting, the ivy.xml is delivered to the CWD, littering the workspace. We # don't need the ivy.xml, so just give it path under the workdir we won't use. "-deliverto", ivyxml + ".unused", "-publish", resolver, "-publishpattern", "{}/[organisation]/[module]/" "[artifact]-[revision](-[classifier]).[ext]".format(self.workdir), "-revision", version, "-m2compatible", ] # TODO(John Sirois): global logging options should be hidden behind some sort of log manager # that we can: # a.) obtain a handle to (dependency injection or manual plumbing) # b.) query for log detail, ie: `if log_manager.is_verbose:` if self.debug: args.append("-verbose") if self.local_snapshot: args.append("-overwrite") try: jvm_options = self._ivy_jvm_options(repo) ivy.execute( jvm_options=jvm_options, args=args, workunit_factory=self.context.new_workunit, workunit_name="ivy-publish", ) except Ivy.Error as e: raise TaskError( f"Failed to push {pushdb_coordinate(jar, entry)}! {e!r}")
def ivy_resolve(self, targets, executor=None, silent=False, workunit_name=None, confs=None, custom_args=None, invalidate_dependents=False): """Resolves external dependencies for the given targets. If there are no targets suitable for jvm transitive dependency resolution, an empty result is returned, ie: ([], {}, None). :param targets: The targets to resolve jvm dependencies for. :type targets: :class:`collections.Iterable` of :class:`pants.build_graph.target.Target` :param executor: A java executor to run ivy with. :type executor: :class:`pants.java.executor.Executor` :param confs: The ivy configurations to resolve; ('default',) by default. :type confs: :class:`collections.Iterable` of string :param custom_args: Any extra command line arguments to pass to ivy. :type custom_args: list of string :param bool invalidate_dependents: `True` to invalidate dependents of targets that needed to be resolved. :returns: A tuple of the classpath, a mapping from ivy cache jars to their linked location under .pants.d, and the id of the reports associated with the resolve. :rtype: tuple of (list, dict, string) """ if not targets: return [], {}, None ivy = Bootstrapper.default_ivy( bootstrap_workunit_factory=self.context.new_workunit) ivy_workdir = os.path.join( self.context.options.for_global_scope().pants_workdir, 'ivy') fingerprint_strategy = IvyResolveFingerprintStrategy(confs) with self.invalidated(targets, invalidate_dependents=invalidate_dependents, silent=silent, fingerprint_strategy=fingerprint_strategy ) as invalidation_check: if not invalidation_check.all_vts: return [], {}, None global_vts = VersionedTargetSet.from_versioned_targets( invalidation_check.all_vts) # If a report file is not present, we need to exec ivy, even if all the individual # targets up to date... See https://rbcommons.com/s/twitter/r/2015 report_missing = False report_confs = confs or ['default'] report_paths = [] resolve_hash_name = global_vts.cache_key.hash for conf in report_confs: report_path = IvyUtils.xml_report_path(self.ivy_cache_dir, resolve_hash_name, conf) if not os.path.exists(report_path): report_missing = True break else: report_paths.append(report_path) target_workdir = os.path.join(ivy_workdir, resolve_hash_name) target_classpath_file = os.path.join(target_workdir, 'classpath') raw_target_classpath_file = target_classpath_file + '.raw' raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp' # A common dir for symlinks into the ivy2 cache. This ensures that paths to jars # in artifact-cached analysis files are consistent across systems. # Note that we have one global, well-known symlink dir, again so that paths are # consistent across builds. symlink_dir = os.path.join(ivy_workdir, 'jars') # Note that it's possible for all targets to be valid but for no classpath file to exist at # target_classpath_file, e.g., if we previously built a superset of targets. if (report_missing or invalidation_check.invalid_vts or not os.path.exists(raw_target_classpath_file)): args = ['-cachepath', raw_target_classpath_file_tmp ] + (custom_args if custom_args else []) self._exec_ivy( target_workdir=target_workdir, targets=global_vts.targets, args=args, executor=executor, ivy=ivy, workunit_name=workunit_name, confs=confs, use_soft_excludes=self.get_options().soft_excludes, resolve_hash_name=resolve_hash_name) if not os.path.exists(raw_target_classpath_file_tmp): raise self.Error( 'Ivy failed to create classpath file at {}'.format( raw_target_classpath_file_tmp)) shutil.move(raw_target_classpath_file_tmp, raw_target_classpath_file) logger.debug('Moved ivy classfile file to {dest}'.format( dest=raw_target_classpath_file)) if self.artifact_cache_writes_enabled(): self.update_artifact_cache([(global_vts, [raw_target_classpath_file])]) else: logger.debug("Using previously resolved reports: {}".format( report_paths)) # Make our actual classpath be symlinks, so that the paths are uniform across systems. # Note that we must do this even if we read the raw_target_classpath_file from the artifact # cache. If we cache the target_classpath_file we won't know how to create the symlinks. with IvyTaskMixin.symlink_map_lock: symlink_map = IvyUtils.symlink_cachepath( self.ivy_cache_dir, raw_target_classpath_file, symlink_dir, target_classpath_file) with IvyUtils.cachepath(target_classpath_file) as classpath: stripped_classpath = [path.strip() for path in classpath] return stripped_classpath, symlink_map, resolve_hash_name
def test_junit_runner(self): # Create the temporary base test directory test_rel_path = 'tests/java/com/pants/foo' test_abs_path = os.path.join(self.build_root, test_rel_path) self.create_dir(test_rel_path) # Generate the temporary java test source code. test_java_file_rel_path = os.path.join(test_rel_path, 'FooTest.java') test_java_file_abs_path = os.path.join(self.build_root, test_java_file_rel_path) self.create_file( test_java_file_rel_path, dedent(''' import org.junit.Test; import static org.junit.Assert.assertTrue; public class FooTest { @Test public void testFoo() { assertTrue(5 > 3); } } ''')) # Invoke ivy to resolve classpath for junit. distribution = Distribution.cached(jdk=True) executor = SubprocessExecutor(distribution=distribution) classpath_file_abs_path = os.path.join(test_abs_path, 'junit.classpath') ivy = Bootstrapper.default_ivy() ivy.execute(args=[ '-cachepath', classpath_file_abs_path, '-dependency', 'junit', 'junit-dep', '4.10' ], executor=executor) with open(classpath_file_abs_path) as fp: classpath = fp.read() # Now directly invoking javac to compile the test java code into java class # so later we can inject the class into products mapping for JUnitRun to execute # the test on. javac = distribution.binary('javac') subprocess.check_call([ javac, '-d', test_abs_path, '-cp', classpath, test_java_file_abs_path ]) # Create a java_tests target and a synthetic resource target. java_tests = self.create_library(test_rel_path, 'java_tests', 'foo_test', ['FooTest.java']) resources = self.make_target('some_resources', Resources) # Set the context with the two targets, one java_tests target and # one synthetic resources target. # The synthetic resources target is to make sure we won't regress # in the future with bug like https://github.com/pantsbuild/pants/issues/508. Note # in that bug, the resources target must be the first one in the list. context = self.context(target_roots=[resources, java_tests]) # Before we run the task, we need to inject the "classes_by_target" with # the compiled test java classes that JUnitRun will know which test # classes to execute. In a normal run, this "classes_by_target" will be # populated by java compiling step. class_products = context.products.get_data( 'classes_by_target', lambda: defaultdict(MultipleRootedProducts)) java_tests_products = MultipleRootedProducts() java_tests_products.add_rel_paths(test_abs_path, ['FooTest.class']) class_products[java_tests] = java_tests_products # Also we need to add the FooTest.class's classpath to the compile_classpath # products data mapping so JUnitRun will be able to add that into the final # classpath under which the junit will be executed. self.populate_compile_classpath(context=context, classpath=[test_abs_path]) # Finally execute the task. self.execute(context)
def ivy_resolve(self, targets, executor=None, symlink_ivyxml=False, silent=False, workunit_name=None, workunit_labels=None): # NOTE: Always pass all the targets to exec_ivy, as they're used to calculate the name of # the generated module, which in turn determines the location of the XML report file # ivy generates. We recompute this name from targets later in order to find that file. # TODO: This is fragile. Refactor so that we're not computing the name twice. if executor and not isinstance(executor, Executor): raise ValueError( 'The executor must be an Executor instance, given %s of type %s' % (executor, type(executor))) ivy = Bootstrapper.default_ivy( java_executor=executor, bootstrap_workunit_factory=self.context.new_workunit) if not targets: return [] ivy_workdir = os.path.join( self.context.config.getdefault('pants_workdir'), 'ivy') ivy_utils = IvyUtils(config=self.context.config, options=self.context.options, log=self.context.log) fingerprint_strategy = IvyResolveFingerprintStrategy() with self.invalidated(targets, invalidate_dependents=True, silent=silent, fingerprint_strategy=fingerprint_strategy ) as invalidation_check: global_vts = VersionedTargetSet.from_versioned_targets( invalidation_check.all_vts) target_workdir = os.path.join(ivy_workdir, global_vts.cache_key.hash) target_classpath_file = os.path.join(target_workdir, 'classpath') raw_target_classpath_file = target_classpath_file + '.raw' raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp' # A common dir for symlinks into the ivy2 cache. This ensures that paths to jars # in artifact-cached analysis files are consistent across systems. # Note that we have one global, well-known symlink dir, again so that paths are # consistent across builds. symlink_dir = os.path.join(ivy_workdir, 'jars') # Note that it's possible for all targets to be valid but for no classpath file to exist at # target_classpath_file, e.g., if we previously built a superset of targets. if invalidation_check.invalid_vts or not os.path.exists( raw_target_classpath_file): args = ['-cachepath', raw_target_classpath_file_tmp] def exec_ivy(): ivy_utils.exec_ivy( target_workdir=target_workdir, targets=targets, args=args, ivy=ivy, workunit_name='ivy', workunit_factory=self.context.new_workunit, symlink_ivyxml=symlink_ivyxml) if workunit_name: with self.context.new_workunit(name=workunit_name, labels=workunit_labels or []): exec_ivy() else: exec_ivy() if not os.path.exists(raw_target_classpath_file_tmp): raise TaskError( 'Ivy failed to create classpath file at %s' % raw_target_classpath_file_tmp) shutil.move(raw_target_classpath_file_tmp, raw_target_classpath_file) logger.debug('Copied ivy classfile file to {dest}'.format( dest=raw_target_classpath_file)) if self.artifact_cache_writes_enabled(): self.update_artifact_cache([(global_vts, [raw_target_classpath_file])]) # Make our actual classpath be symlinks, so that the paths are uniform across systems. # Note that we must do this even if we read the raw_target_classpath_file from the artifact # cache. If we cache the target_classpath_file we won't know how to create the symlinks. symlink_map = IvyUtils.symlink_cachepath(self.context.ivy_home, raw_target_classpath_file, symlink_dir, target_classpath_file) with IvyTaskMixin.symlink_map_lock: all_symlinks_map = self.context.products.get_data( 'symlink_map') or defaultdict(list) for path, symlink in symlink_map.items(): all_symlinks_map[os.path.realpath(path)].append(symlink) self.context.products.safe_create_data('symlink_map', lambda: all_symlinks_map) with IvyUtils.cachepath(target_classpath_file) as classpath: stripped_classpath = [path.strip() for path in classpath] return [ path for path in stripped_classpath if ivy_utils.is_classpath_artifact(path) ]
def ivy_resolve(self, targets, executor=None, silent=False, workunit_name=None, confs=None, custom_args=None): """Populates the product 'ivy_resolve_symlink_map' from the specified targets.""" if not targets: return ([], set()) # NOTE: Always pass all the targets to exec_ivy, as they're used to calculate the name of # the generated module, which in turn determines the location of the XML report file # ivy generates. We recompute this name from targets later in order to find that file. # TODO: This is fragile. Refactor so that we're not computing the name twice. ivy = Bootstrapper.default_ivy(bootstrap_workunit_factory=self.context.new_workunit) ivy_workdir = os.path.join(self.context.options.for_global_scope().pants_workdir, 'ivy') fingerprint_strategy = IvyResolveFingerprintStrategy() with self.invalidated(targets, invalidate_dependents=False, silent=silent, fingerprint_strategy=fingerprint_strategy) as invalidation_check: if not invalidation_check.all_vts: return ([], set()) global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts) target_workdir = os.path.join(ivy_workdir, global_vts.cache_key.hash) target_classpath_file = os.path.join(target_workdir, 'classpath') raw_target_classpath_file = target_classpath_file + '.raw' raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp' # A common dir for symlinks into the ivy2 cache. This ensures that paths to jars # in artifact-cached analysis files are consistent across systems. # Note that we have one global, well-known symlink dir, again so that paths are # consistent across builds. symlink_dir = os.path.join(ivy_workdir, 'jars') # Note that it's possible for all targets to be valid but for no classpath file to exist at # target_classpath_file, e.g., if we previously built a superset of targets. if invalidation_check.invalid_vts or not os.path.exists(raw_target_classpath_file): args = ['-cachepath', raw_target_classpath_file_tmp] + (custom_args if custom_args else []) self.exec_ivy( target_workdir=target_workdir, targets=global_vts.targets, args=args, executor=executor, ivy=ivy, workunit_name=workunit_name, confs=confs) if not os.path.exists(raw_target_classpath_file_tmp): raise TaskError('Ivy failed to create classpath file at %s' % raw_target_classpath_file_tmp) shutil.move(raw_target_classpath_file_tmp, raw_target_classpath_file) logger.debug('Copied ivy classfile file to {dest}'.format(dest=raw_target_classpath_file)) if self.artifact_cache_writes_enabled(): self.update_artifact_cache([(global_vts, [raw_target_classpath_file])]) # Make our actual classpath be symlinks, so that the paths are uniform across systems. # Note that we must do this even if we read the raw_target_classpath_file from the artifact # cache. If we cache the target_classpath_file we won't know how to create the symlinks. symlink_map = IvyUtils.symlink_cachepath(ivy.ivy_cache_dir, raw_target_classpath_file, symlink_dir, target_classpath_file) with IvyTaskMixin.symlink_map_lock: products = self.context.products all_symlinks_map = products.get_data('ivy_resolve_symlink_map') or defaultdict(list) for path, symlink in symlink_map.items(): all_symlinks_map[os.path.realpath(path)].append(symlink) products.safe_create_data('ivy_resolve_symlink_map', lambda: all_symlinks_map) with IvyUtils.cachepath(target_classpath_file) as classpath: stripped_classpath = [path.strip() for path in classpath] return (stripped_classpath, global_vts.targets)
def test_junit_runner(self): # Create the temporary base test directory test_rel_path = 'tests/java/com/pants/foo' test_abs_path = os.path.join(self.build_root, test_rel_path) self.create_dir(test_rel_path) # Generate the temporary java test source code. test_java_file_rel_path = os.path.join(test_rel_path, 'FooTest.java') test_java_file_abs_path = os.path.join(self.build_root, test_java_file_rel_path) self.create_file(test_java_file_rel_path, dedent(''' import org.junit.Test; import static org.junit.Assert.assertTrue; public class FooTest { @Test public void testFoo() { assertTrue(5 > 3); } } ''')) # Invoke ivy to resolve classpath for junit. distribution = Distribution.cached(jdk=True) executor = SubprocessExecutor(distribution=distribution) classpath_file_abs_path = os.path.join(test_abs_path, 'junit.classpath') ivy = Bootstrapper.default_ivy(java_executor=executor) ivy.execute(args=['-cachepath', classpath_file_abs_path, '-dependency', 'junit', 'junit-dep', '4.10']) with open(classpath_file_abs_path) as fp: classpath = fp.read() # Now directly invoking javac to compile the test java code into java class # so later we can inject the class into products mapping for JUnitRun to execute # the test on. javac = distribution.binary('javac') subprocess.check_call( [javac, '-d', test_abs_path, '-cp', classpath, test_java_file_abs_path]) # Create a java_tests target and a synthetic resource target. java_tests = self.create_library(test_rel_path, 'java_tests', 'foo_test', ['FooTest.java']) resources = self.make_target('some_resources', Resources) # Set the context with the two targets, one java_tests target and # one synthetic resources target. # The synthetic resources target is to make sure we won't regress # in the future with bug like https://github.com/pantsbuild/pants/issues/508. Note # in that bug, the resources target must be the first one in the list. context = self.context(target_roots=[resources, java_tests]) # Before we run the task, we need to inject the "classes_by_target" with # the compiled test java classes that JUnitRun will know which test # classes to execute. In a normal run, this "classes_by_target" will be # populated by java compiling step. class_products = context.products.get_data( 'classes_by_target', lambda: defaultdict(MultipleRootedProducts)) java_tests_products = MultipleRootedProducts() java_tests_products.add_rel_paths(test_abs_path, ['FooTest.class']) class_products[java_tests] = java_tests_products # Also we need to add the FooTest.class's classpath to the exclusive_groups # products data mapping so JUnitRun will be able to add that into the final # classpath under which the junit will be executed. self.populate_exclusive_groups( context=context, classpaths=[test_abs_path], # This is a bit hacky. The issue in https://github.com/pantsbuild/pants/issues/508 # is that normal resources specified in the BUILD targets are fine, but the # synthetic resources ones generated on the fly don't have exclusive_groups data # mapping entries thus later on in _JUnitRunner lookup it blows up. So we manually # exclude the synthetic resources target here to simulate that situation and ensure # the _JUnitRunner does filter out all the non-java-tests targets. target_predicate=lambda t: t != resources) # Finally execute the task. self.execute(context)
def test_default_ivy(self): ivy = Bootstrapper.default_ivy() self.assertIsNotNone(ivy.ivy_cache_dir) self.assertIsNone(ivy.ivy_settings)
def test_default_ivy(self): with subsystem_instance(IvySubsystem): ivy = Bootstrapper.default_ivy() self.assertIsNotNone(ivy.ivy_cache_dir) self.assertIsNone(ivy.ivy_settings)