def test_fresh_bootstrap(self):
with temporary_dir() as fresh_bootstrap_dir:
self.set_bootstrap_options(pants_bootstrapdir=fresh_bootstrap_dir)
# Initialize the Ivy subsystem
self.context()
Bootstrapper.default_ivy()
bootstrap_jar_path = os.path.join(fresh_bootstrap_dir,
'tools', 'jvm', 'ivy', 'bootstrap.jar')
self.assertTrue(os.path.exists(bootstrap_jar_path))
def publish(self, ivyxml_path, jar, entry, repo, published):
"""Run ivy to publish a jar. ivyxml_path is the path to the ivy file; published
is a list of jars published so far (including this one). entry is a pushdb entry."""
jvm_args = self._ivy_jvm_args(repo)
resolver = repo['resolver']
path = repo.get('path')
try:
ivy = Bootstrapper.default_ivy()
except Bootstrapper.Error as e:
raise TaskError('Failed to push {0}! {1}'.format(pushdb_coordinate(jar, entry), e))
ivysettings = self.generate_ivysettings(ivy, published, publish_local=path)
args = [
'-settings', ivysettings,
'-ivy', ivyxml_path,
'-deliverto', '%s/[organisation]/[module]/ivy-[revision].xml' % self.workdir,
'-publish', resolver,
'-publishpattern', '%s/[organisation]/[module]/'
'[artifact]-[revision](-[classifier]).[ext]' % self.workdir,
'-revision', entry.version().version(),
'-m2compatible',
]
if LogOptions.stderr_log_level() == logging.DEBUG:
args.append('-verbose')
if self.local_snapshot:
args.append('-overwrite')
try:
ivy.execute(jvm_options=jvm_args, args=args,
workunit_factory=self.context.new_workunit, workunit_name='jar-publish')
except Ivy.Error as e:
raise TaskError('Failed to push {0}! {1}'.format(pushdb_coordinate(jar, entry), e))
def exec_ivy(cls, ivy, confs, ivyxml, args,
jvm_options,
executor,
workunit_name,
workunit_factory):
"""
:API: public
"""
ivy = ivy or Bootstrapper.default_ivy()
ivy_args = ['-ivy', ivyxml]
ivy_args.append('-confs')
ivy_args.extend(confs)
ivy_args.extend(args)
ivy_jvm_options = list(jvm_options)
# Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng.
ivy_jvm_options.append('-Dsun.io.useCanonCaches=false')
runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor)
try:
with ivy.resolution_lock:
result = execute_runner(runner, workunit_factory=workunit_factory,
workunit_name=workunit_name)
if result != 0:
raise IvyUtils.IvyError('Ivy returned {result}. cmd={cmd}'.format(result=result, cmd=runner.cmd))
except runner.executor.Error as e:
raise IvyUtils.IvyError(e)
def exec_ivy(cls, ivy, confs, ivyxml, args, jvm_options, executor,
workunit_name, workunit_factory):
ivy = ivy or Bootstrapper.default_ivy()
ivy_args = ['-ivy', ivyxml]
ivy_args.append('-confs')
ivy_args.extend(confs)
ivy_args.extend(args)
ivy_jvm_options = list(jvm_options)
# Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng.
ivy_jvm_options.append('-Dsun.io.useCanonCaches=false')
runner = ivy.runner(jvm_options=ivy_jvm_options,
args=ivy_args,
executor=executor)
try:
result = execute_runner(runner,
workunit_factory=workunit_factory,
workunit_name=workunit_name)
if result != 0:
raise IvyUtils.IvyError(
'Ivy returned {result}. cmd={cmd}'.format(result=result,
cmd=runner.cmd))
except runner.executor.Error as e:
raise IvyUtils.IvyError(e)
def publish(ivyxml_path):
ivysettings = self.generate_ivysettings(published, publish_local=path)
args = [
'-settings', ivysettings,
'-ivy', ivyxml_path,
'-deliverto', '%s/[organisation]/[module]/ivy-[revision].xml' % self.workdir,
'-publish', resolver,
'-publishpattern', '%s/[organisation]/[module]/'
'[artifact]-[revision](-[classifier]).[ext]' % self.workdir,
'-revision', newver.version(),
'-m2compatible',
]
if LogOptions.stderr_log_level() == logging.DEBUG:
args.append('-verbose')
if self.snapshot:
args.append('-overwrite')
try:
ivy = Bootstrapper.default_ivy()
ivy.execute(jvm_options=jvm_args, args=args,
workunit_factory=self.context.new_workunit, workunit_name='jar-publish')
except (Bootstrapper.Error, Ivy.Error) as e:
raise TaskError('Failed to push %s! %s' % (jar_coordinate(jar, newver.version()), e))
def publish(ivyxml_path):
try:
ivy = Bootstrapper.default_ivy()
except Bootstrapper.Error as e:
raise TaskError('Failed to push %s! %s' % (jar_coordinate(jar, newver.version()), e))
ivysettings = self.generate_ivysettings(ivy, published, publish_local=path)
args = [
'-settings', ivysettings,
'-ivy', ivyxml_path,
'-deliverto', '%s/[organisation]/[module]/ivy-[revision].xml' % self.workdir,
'-publish', resolver,
'-publishpattern', '%s/[organisation]/[module]/'
'[artifact]-[revision](-[classifier]).[ext]' % self.workdir,
'-revision', newver.version(),
'-m2compatible',
]
if LogOptions.stderr_log_level() == logging.DEBUG:
args.append('-verbose')
if self.snapshot:
args.append('-overwrite')
try:
ivy.execute(jvm_options=jvm_args, args=args,
workunit_factory=self.context.new_workunit, workunit_name='jar-publish')
except Ivy.Error as e:
raise TaskError('Failed to push %s! %s' % (jar_coordinate(jar, newver.version()), e))
def execute_junit_runner(self, content):
# Create the temporary base test directory
test_rel_path = 'tests/java/org/pantsbuild/foo'
test_abs_path = os.path.join(self.build_root, test_rel_path)
self.create_dir(test_rel_path)
# Generate the temporary java test source code.
test_java_file_rel_path = os.path.join(test_rel_path, 'FooTest.java')
test_java_file_abs_path = os.path.join(self.build_root, test_java_file_rel_path)
self.create_file(test_java_file_rel_path, content)
# Invoke ivy to resolve classpath for junit.
distribution = Distribution.cached(jdk=True)
executor = SubprocessExecutor(distribution=distribution)
classpath_file_abs_path = os.path.join(test_abs_path, 'junit.classpath')
ivy = Bootstrapper.default_ivy()
ivy.execute(args=['-cachepath', classpath_file_abs_path,
'-dependency', 'junit', 'junit-dep', '4.10'], executor=executor)
with open(classpath_file_abs_path) as fp:
classpath = fp.read()
# Now directly invoking javac to compile the test java code into java class
# so later we can inject the class into products mapping for JUnitRun to execute
# the test on.
javac = distribution.binary('javac')
subprocess.check_call(
[javac, '-d', test_abs_path, '-cp', classpath, test_java_file_abs_path])
# Create a java_tests target and a synthetic resource target.
java_tests = self.create_library(test_rel_path, 'java_tests', 'foo_test', ['FooTest.java'])
resources = self.make_target('some_resources', Resources)
# Set the context with the two targets, one java_tests target and
# one synthetic resources target.
# The synthetic resources target is to make sure we won't regress
# in the future with bug like https://github.com/pantsbuild/pants/issues/508. Note
# in that bug, the resources target must be the first one in the list.
context = self.context(target_roots=[resources, java_tests])
# Before we run the task, we need to inject the "classes_by_target" with
# the compiled test java classes that JUnitRun will know which test
# classes to execute. In a normal run, this "classes_by_target" will be
# populated by java compiling step.
class_products = context.products.get_data(
'classes_by_target', lambda: defaultdict(MultipleRootedProducts))
java_tests_products = MultipleRootedProducts()
java_tests_products.add_rel_paths(test_abs_path, ['FooTest.class'])
class_products[java_tests] = java_tests_products
# Also we need to add the FooTest.class's classpath to the compile_classpath
# products data mapping so JUnitRun will be able to add that into the final
# classpath under which the junit will be executed.
self.populate_compile_classpath(
context=context,
classpath=[test_abs_path])
# Finally execute the task.
self.execute(context)
def exec_ivy(self,
target_workdir,
targets,
args,
confs=None,
ivy=None,
workunit_name='ivy',
workunit_factory=None,
symlink_ivyxml=False,
jars=None):
ivy = ivy or Bootstrapper.default_ivy()
if not isinstance(ivy, Ivy):
raise ValueError('The ivy argument supplied must be an Ivy instance, given %s of type %s'
% (ivy, type(ivy)))
ivyxml = os.path.join(target_workdir, 'ivy.xml')
if not jars:
jars, excludes = self._calculate_classpath(targets)
else:
excludes = set()
ivy_args = ['-ivy', ivyxml]
confs_to_resolve = confs or ['default']
ivy_args.append('-confs')
ivy_args.extend(confs_to_resolve)
ivy_args.extend(args)
if not self._transitive:
ivy_args.append('-notransitive')
ivy_args.extend(self._args)
def safe_link(src, dest):
try:
os.unlink(dest)
except OSError as e:
if e.errno != errno.ENOENT:
raise
os.symlink(src, dest)
with IvyUtils.ivy_lock:
self._generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve)
runner = ivy.runner(jvm_options=self._jvm_options, args=ivy_args)
try:
result = util.execute_runner(runner,
workunit_factory=workunit_factory,
workunit_name=workunit_name)
# Symlink to the current ivy.xml file (useful for IDEs that read it).
if symlink_ivyxml:
ivyxml_symlink = os.path.join(self._workdir, 'ivy.xml')
safe_link(ivyxml, ivyxml_symlink)
if result != 0:
raise TaskError('Ivy returned %d' % result)
except runner.executor.Error as e:
raise TaskError(e)
def publish(self, publications, jar, entry, repo, published):
"""Run ivy to publish a jar. ivyxml_path is the path to the ivy file; published
is a list of jars published so far (including this one). entry is a pushdb entry."""
try:
ivy = Bootstrapper.default_ivy()
except Bootstrapper.Error as e:
raise TaskError('Failed to push {0}! {1}'.format(
pushdb_coordinate(jar, entry), e))
path = repo.get('path')
ivysettings = self.generate_ivysettings(ivy,
published,
publish_local=path)
version = entry.version().version()
ivyxml = self.generate_ivy(jar, version, publications)
resolver = repo['resolver']
args = [
'-settings',
ivysettings,
'-ivy',
ivyxml,
# Without this setting, the ivy.xml is delivered to the CWD, littering the workspace. We
# don't need the ivy.xml, so just give it path under the workdir we won't use.
'-deliverto',
ivyxml + '.unused',
'-publish',
resolver,
'-publishpattern',
'{}/[organisation]/[module]/'
'[artifact]-[revision](-[classifier]).[ext]'.format(self.workdir),
'-revision',
version,
'-m2compatible',
]
# TODO(John Sirois): global logging options should be hidden behind some sort of log manager
# that we can:
# a.) obtain a handle to (dependency injection or manual plumbing)
# b.) query for log detail, ie: `if log_manager.is_verbose:`
if self.get_options().level == 'debug':
args.append('-verbose')
if self.local_snapshot:
args.append('-overwrite')
try:
jvm_options = self._ivy_jvm_options(repo)
ivy.execute(jvm_options=jvm_options,
args=args,
workunit_factory=self.context.new_workunit,
workunit_name='ivy-publish')
except Ivy.Error as e:
raise TaskError('Failed to push {0}! {1}'.format(
pushdb_coordinate(jar, entry), e))
def publish(self, publications, jar, entry, repo, published):
"""Run ivy to publish a jar. ivyxml_path is the path to the ivy file; published
is a list of jars published so far (including this one). entry is a pushdb entry."""
try:
ivy = Bootstrapper.default_ivy()
except Bootstrapper.Error as e:
raise TaskError("Failed to push {0}! {1}".format(pushdb_coordinate(jar, entry), e))
path = repo.get("path")
ivysettings = self.generate_ivysettings(ivy, published, publish_local=path)
version = entry.version().version()
ivyxml = self.generate_ivy(jar, version, publications)
resolver = repo["resolver"]
args = [
"-settings",
ivysettings,
"-ivy",
ivyxml,
# Without this setting, the ivy.xml is delivered to the CWD, littering the workspace. We
# don't need the ivy.xml, so just give it path under the workdir we won't use.
"-deliverto",
ivyxml + ".unused",
"-publish",
resolver,
"-publishpattern",
"{}/[organisation]/[module]/" "[artifact]-[revision](-[classifier]).[ext]".format(self.workdir),
"-revision",
version,
"-m2compatible",
]
# TODO(John Sirois): global logging options should be hidden behind some sort of log manager
# that we can:
# a.) obtain a handle to (dependency injection or manual plumbing)
# b.) query for log detail, ie: `if log_manager.is_verbose:`
if self.get_options().level == "debug":
args.append("-verbose")
if self.local_snapshot:
args.append("-overwrite")
try:
jvm_options = self._ivy_jvm_options(repo)
ivy.execute(
jvm_options=jvm_options,
args=args,
workunit_factory=self.context.new_workunit,
workunit_name="ivy-publish",
)
except Ivy.Error as e:
raise TaskError("Failed to push {0}! {1}".format(pushdb_coordinate(jar, entry), e))
def mapjars(self,
genmap,
target,
executor,
workunit_factory=None,
jars=None):
"""Resolves jars for the target and stores their locations in genmap.
:param genmap: The jar_dependencies ProductMapping entry for the required products.
:param target: The target whose jar dependencies are being retrieved.
:param jars: If specified, resolves the given jars rather than
:type jars: List of :class:`pants.backend.jvm.targets.jar_dependency.JarDependency` (jar())
objects.
"""
mapdir = os.path.join(self.mapto_dir(), target.id)
safe_mkdir(mapdir, clean=True)
ivyargs = [
'-retrieve',
'%s/[organisation]/[artifact]/[conf]/'
'[organisation]-[artifact]-[revision](-[classifier]).[ext]' %
mapdir,
'-symlink',
]
self.exec_ivy(mapdir, [target],
ivyargs,
confs=target.payload.configurations,
ivy=Bootstrapper.default_ivy(executor),
workunit_factory=workunit_factory,
workunit_name='map-jars',
jars=jars)
for org in os.listdir(mapdir):
orgdir = os.path.join(mapdir, org)
if os.path.isdir(orgdir):
for name in os.listdir(orgdir):
artifactdir = os.path.join(orgdir, name)
if os.path.isdir(artifactdir):
for conf in os.listdir(artifactdir):
confdir = os.path.join(artifactdir, conf)
for f in os.listdir(confdir):
if self.is_mappable_artifact(org, name, f):
# TODO(John Sirois): kill the org and (org, name) exclude mappings in favor of a
# conf whitelist
genmap.add(org, confdir).append(f)
genmap.add((org, name), confdir).append(f)
genmap.add(target, confdir).append(f)
genmap.add((target, conf),
confdir).append(f)
genmap.add((org, name, conf),
confdir).append(f)
def publish(self, publications, jar, version, repo, published):
"""Run ivy to publish a jar.
:param str ivyxml_path: The path to the ivy file.
:param list published: A list of jars published so far (including this one).
"""
try:
ivy = Bootstrapper.default_ivy()
except Bootstrapper.Error as e:
raise TaskError('Failed to push {0}! {1}'.format(jar, e))
if ivy.ivy_settings is None:
raise TaskError('An ivysettings.xml with writeable resolvers is required for publishing, '
'but none was configured.')
path = repo.get('path')
ivysettings = self.generate_ivysettings(self.fetch_ivysettings(ivy), published, publish_local=path)
ivyxml = self.generate_ivy(jar, version, publications)
resolver = repo['resolver']
args = [
'-settings', ivysettings,
'-ivy', ivyxml,
# Without this setting, the ivy.xml is delivered to the CWD, littering the workspace. We
# don't need the ivy.xml, so just give it path under the workdir we won't use.
'-deliverto', ivyxml + '.unused',
'-publish', resolver,
'-publishpattern', '{}/[organisation]/[module]/'
'[artifact]-[revision](-[classifier]).[ext]'.format(self.workdir),
'-revision', version,
'-m2compatible',
]
if self.get_options().level == 'debug':
args.append('-verbose')
if self.get_options().local:
args.append('-overwrite')
if not self.get_options().dryrun:
try:
ivy.execute(jvm_options=self._ivy_jvm_options(repo), args=args,
workunit_factory=self.context.new_workunit, workunit_name='ivy-publish')
except Ivy.Error as e:
raise TaskError('Failed to push {0}! {1}'.format(jar, e))
else:
print("\nDRYRUN- would have pushed:{} using {} resolver.".format(self.jar_coordinate(jar, version), resolver))
def _exec_ivy(self,
target_workdir,
targets,
args,
executor=None,
confs=None,
ivy=None,
workunit_name='ivy',
use_soft_excludes=False,
resolve_hash_name=None):
ivy_jvm_options = self.get_options().jvm_options[:]
# Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng.
ivy_jvm_options.append('-Dsun.io.useCanonCaches=false')
ivy = ivy or Bootstrapper.default_ivy()
ivyxml = os.path.join(target_workdir, 'ivy.xml')
ivy_args = ['-ivy', ivyxml]
confs_to_resolve = confs or ('default', )
ivy_args.append('-confs')
ivy_args.extend(confs_to_resolve)
ivy_args.extend(args)
# TODO(John Sirois): merge the code below into IvyUtils or up here; either way, better
# diagnostics can be had in `IvyUtils.generate_ivy` if this is done.
# See: https://github.com/pantsbuild/pants/issues/2239
try:
jars, excludes = IvyUtils.calculate_classpath(
targets, gather_excludes=not use_soft_excludes)
with IvyUtils.ivy_lock:
IvyUtils.generate_ivy(targets, jars, excludes, ivyxml,
confs_to_resolve, resolve_hash_name)
runner = ivy.runner(jvm_options=ivy_jvm_options,
args=ivy_args,
executor=executor)
try:
result = execute_runner(
runner,
workunit_factory=self.context.new_workunit,
workunit_name=workunit_name)
if result != 0:
raise self.Error(
'Ivy returned {result}. cmd={cmd}'.format(
result=result, cmd=runner.cmd))
except runner.executor.Error as e:
raise self.Error(e)
except IvyUtils.IvyError as e:
raise self.Error('Failed to prepare ivy resolve: {}'.format(e))
def exec_ivy(self,
target_workdir,
targets,
args,
executor=None,
confs=None,
ivy=None,
workunit_name='ivy',
jars=None,
use_soft_excludes=False,
resolve_hash_name=None):
ivy_jvm_options = copy.copy(self.get_options().jvm_options)
# Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng.
ivy_jvm_options.append('-Dsun.io.useCanonCaches=false')
ivy = ivy or Bootstrapper.default_ivy()
ivyxml = os.path.join(target_workdir, 'ivy.xml')
if not jars:
jars, excludes = IvyUtils.calculate_classpath(
targets, gather_excludes=not use_soft_excludes)
else:
excludes = set()
ivy_args = ['-ivy', ivyxml]
confs_to_resolve = confs or ['default']
ivy_args.append('-confs')
ivy_args.extend(confs_to_resolve)
ivy_args.extend(args)
with IvyUtils.ivy_lock:
IvyUtils.generate_ivy(targets, jars, excludes, ivyxml,
confs_to_resolve, resolve_hash_name)
runner = ivy.runner(jvm_options=ivy_jvm_options,
args=ivy_args,
executor=executor)
try:
result = execute_runner(
runner,
workunit_factory=self.context.new_workunit,
workunit_name=workunit_name)
if result != 0:
raise TaskError('Ivy returned {result}. cmd={cmd}'.format(
result=result, cmd=runner.cmd))
except runner.executor.Error as e:
raise TaskError(e)
def run_antlrs(self, output_dir):
args = [
'-dependency', 'org.antlr', 'antlr', self.target.antlr_version,
'-types', 'jar', '-main', 'org.antlr.Tool', '--', '-fo', output_dir
]
for source in self.target.sources_relative_to_buildroot():
abs_path = os.path.join(get_buildroot(), source)
args.append(abs_path)
try:
ivy = Bootstrapper.default_ivy()
ivy.execute(
args=args
) # TODO: Needs a workunit, when we have a context here.
return True
except (Bootstrapper.Error, Ivy.Error) as e:
print('ANTLR generation failed! %s' % e, file=sys.stderr)
return False
def run_antlrs(self, output_dir):
# TODO(John Sirois): graduate to a JvmToolTask and/or merge with the java code gen AntlrGen
# task.
args = [
'-dependency', 'org.antlr', 'antlr', self.target.antlr_version,
'-types', 'jar',
'-main', 'org.antlr.Tool',
'--', '-fo', output_dir
]
for source in self.target.sources_relative_to_buildroot():
abs_path = os.path.join(get_buildroot(), source)
args.append(abs_path)
try:
ivy = Bootstrapper.default_ivy()
ivy.execute(args=args) # TODO: Needs a workunit, when we have a context here.
except (Bootstrapper.Error, Ivy.Error) as e:
raise TaskError('ANTLR generation failed! {0}'.format(e))
def run_antlrs(self, output_dir):
# TODO(John Sirois): graduate to a JvmToolTask and/or merge with the java code gen AntlrGen
# task.
args = [
'-dependency', 'org.antlr', 'antlr', self.target.antlr_version,
'-types', 'jar', '-main', 'org.antlr.Tool', '--', '-fo', output_dir
]
for source in self.target.sources_relative_to_buildroot():
abs_path = os.path.join(get_buildroot(), source)
args.append(abs_path)
try:
ivy = Bootstrapper.default_ivy()
ivy.execute(
args=args
) # TODO: Needs a workunit, when we have a context here.
except (Bootstrapper.Error, Ivy.Error) as e:
raise TaskError('ANTLR generation failed! {0}'.format(e))
def _exec_ivy(
self,
target_workdir,
targets,
args,
executor=None,
confs=None,
ivy=None,
workunit_name="ivy",
use_soft_excludes=False,
resolve_hash_name=None,
):
ivy_jvm_options = self.get_options().jvm_options[:]
# Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng.
ivy_jvm_options.append("-Dsun.io.useCanonCaches=false")
ivy = ivy or Bootstrapper.default_ivy()
ivyxml = os.path.join(target_workdir, "ivy.xml")
ivy_args = ["-ivy", ivyxml]
confs_to_resolve = confs or ("default",)
ivy_args.append("-confs")
ivy_args.extend(confs_to_resolve)
ivy_args.extend(args)
# TODO(John Sirois): merge the code below into IvyUtils or up here; either way, better
# diagnostics can be had in `IvyUtils.generate_ivy` if this is done.
# See: https://github.com/pantsbuild/pants/issues/2239
try:
jars, excludes = IvyUtils.calculate_classpath(targets, gather_excludes=not use_soft_excludes)
with IvyUtils.ivy_lock:
IvyUtils.generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve, resolve_hash_name)
runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor)
try:
result = execute_runner(
runner, workunit_factory=self.context.new_workunit, workunit_name=workunit_name
)
if result != 0:
raise self.Error("Ivy returned {result}. cmd={cmd}".format(result=result, cmd=runner.cmd))
except runner.executor.Error as e:
raise self.Error(e)
except IvyUtils.IvyError as e:
raise self.Error("Failed to prepare ivy resolve: {}".format(e))
def exec_ivy(self,
target_workdir,
targets,
args,
executor=None,
confs=None,
ivy=None,
workunit_name='ivy',
jars=None,
use_soft_excludes=False,
resolve_hash_name=None):
ivy_jvm_options = copy.copy(self.get_options().jvm_options)
# Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng.
ivy_jvm_options.append('-Dsun.io.useCanonCaches=false')
ivy = ivy or Bootstrapper.default_ivy()
ivyxml = os.path.join(target_workdir, 'ivy.xml')
if not jars:
automatic_excludes = self.get_options().automatic_excludes
jars, excludes = IvyUtils.calculate_classpath(targets,
gather_excludes=not use_soft_excludes,
automatic_excludes=automatic_excludes)
else:
excludes = set()
ivy_args = ['-ivy', ivyxml]
confs_to_resolve = confs or ['default']
ivy_args.append('-confs')
ivy_args.extend(confs_to_resolve)
ivy_args.extend(args)
with IvyUtils.ivy_lock:
IvyUtils.generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve, resolve_hash_name)
runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor)
try:
result = execute_runner(runner, workunit_factory=self.context.new_workunit,
workunit_name=workunit_name)
if result != 0:
raise TaskError('Ivy returned {result}. cmd={cmd}'.format(result=result, cmd=runner.cmd))
except runner.executor.Error as e:
raise TaskError(e)
def mapjars(self, genmap, target, executor, jars=None):
"""Resolves jars for the target and stores their locations in genmap.
:param genmap: The jar_dependencies ProductMapping entry for the required products.
:param target: The target whose jar dependencies are being retrieved.
:param jars: If specified, resolves the given jars rather than
:type jars: List of :class:`pants.backend.jvm.targets.jar_dependency.JarDependency` (jar())
objects.
"""
mapdir = os.path.join(self.workdir, 'mapped-jars', target.id)
safe_mkdir(mapdir, clean=True)
ivyargs = [
'-retrieve', '%s/[organisation]/[artifact]/[conf]/'
'[organisation]-[artifact]-[revision](-[classifier]).[ext]' % mapdir,
'-symlink',
]
confs = maybe_list(target.payload.get_field_value('configurations') or [])
self.exec_ivy(mapdir,
[target],
executor=executor,
args=ivyargs,
confs=confs,
ivy=Bootstrapper.default_ivy(),
workunit_name='map-jars',
jars=jars)
for org in os.listdir(mapdir):
orgdir = os.path.join(mapdir, org)
if os.path.isdir(orgdir):
for name in os.listdir(orgdir):
artifactdir = os.path.join(orgdir, name)
if os.path.isdir(artifactdir):
for conf in os.listdir(artifactdir):
confdir = os.path.join(artifactdir, conf)
for f in os.listdir(confdir):
# TODO(John Sirois): kill the org and (org, name) exclude mappings in favor of a
# conf whitelist
genmap.add(org, confdir).append(f)
genmap.add((org, name), confdir).append(f)
genmap.add(target, confdir).append(f)
genmap.add((target, conf), confdir).append(f)
genmap.add((org, name, conf), confdir).append(f)
def do_resolve(cls, executor, extra_args, ivyxml, jvm_options, workdir_report_paths_by_conf,
confs, ivy_cache_dir, ivy_cache_classpath_filename, resolve_hash_name,
workunit_factory, workunit_name):
"""Execute Ivy with the given ivy.xml and copies all relevant files into the workdir.
This method does an Ivy resolve, which may be either a Pants resolve or a Pants fetch depending
on whether there is an existing frozen resolution.
After it is run, the Ivy reports are copied into the workdir at the paths specified by
workdir_report_paths_by_conf along with a file containing a list of all the requested artifacts
and their transitive dependencies.
:param executor: A JVM executor to use to invoke ivy.
:param extra_args: Extra arguments to pass to ivy.
:param ivyxml: The input ivy.xml containing the dependencies to resolve.
:param jvm_options: A list of jvm option strings to use for the ivy invoke, or None.
:param workdir_report_paths_by_conf: A dict mapping confs to report paths in the workdir.
:param confs: The confs used in the resolve.
:param resolve_hash_name: The hash to use as the module name for finding the ivy report file.
:param workunit_factory: A workunit factory for the ivy invoke, or None.
:param workunit_name: A workunit name for the ivy invoke, or None.
"""
ivy = Bootstrapper.default_ivy(bootstrap_workunit_factory=workunit_factory)
with safe_concurrent_creation(ivy_cache_classpath_filename) as raw_target_classpath_file_tmp:
extra_args = extra_args or []
args = ['-cachepath', raw_target_classpath_file_tmp] + extra_args
with cls._ivy_lock:
cls._exec_ivy(ivy, confs, ivyxml, args,
jvm_options=jvm_options,
executor=executor,
workunit_name=workunit_name,
workunit_factory=workunit_factory)
if not os.path.exists(raw_target_classpath_file_tmp):
raise cls.IvyError('Ivy failed to create classpath file at {}'
.format(raw_target_classpath_file_tmp))
cls._copy_ivy_reports(workdir_report_paths_by_conf, confs, ivy_cache_dir, resolve_hash_name)
logger.debug('Moved ivy classfile file to {dest}'
.format(dest=ivy_cache_classpath_filename))
def mapjars(self, genmap, target, executor, workunit_factory=None):
"""
Parameters:
genmap: the jar_dependencies ProductMapping entry for the required products.
target: the target whose jar dependencies are being retrieved.
"""
mapdir = os.path.join(self.mapto_dir(), target.id)
safe_mkdir(mapdir, clean=True)
ivyargs = [
'-retrieve',
'%s/[organisation]/[artifact]/[conf]/'
'[organisation]-[artifact]-[revision](-[classifier]).[ext]' %
mapdir,
'-symlink',
]
self.exec_ivy(mapdir, [target],
ivyargs,
confs=target.payload.configurations,
ivy=Bootstrapper.default_ivy(executor),
workunit_factory=workunit_factory,
workunit_name='map-jars')
for org in os.listdir(mapdir):
orgdir = os.path.join(mapdir, org)
if os.path.isdir(orgdir):
for name in os.listdir(orgdir):
artifactdir = os.path.join(orgdir, name)
if os.path.isdir(artifactdir):
for conf in os.listdir(artifactdir):
confdir = os.path.join(artifactdir, conf)
for f in os.listdir(confdir):
if self.is_mappable_artifact(org, name, f):
# TODO(John Sirois): kill the org and (org, name) exclude mappings in favor of a
# conf whitelist
genmap.add(org, confdir).append(f)
genmap.add((org, name), confdir).append(f)
genmap.add(target, confdir).append(f)
genmap.add((target, conf),
confdir).append(f)
genmap.add((org, name, conf),
confdir).append(f)
def publish(self, ivyxml_path, jar, entry, repo, published):
"""Run ivy to publish a jar. ivyxml_path is the path to the ivy file; published
is a list of jars published so far (including this one). entry is a pushdb entry."""
jvm_options = self._ivy_jvm_options(repo)
resolver = repo['resolver']
path = repo.get('path')
try:
ivy = Bootstrapper.default_ivy()
except Bootstrapper.Error as e:
raise TaskError('Failed to push {0}! {1}'.format(pushdb_coordinate(jar, entry), e))
ivysettings = self.generate_ivysettings(ivy, published, publish_local=path)
args = [
'-settings', ivysettings,
'-ivy', ivyxml_path,
'-deliverto', '%s/[organisation]/[module]/ivy-[revision].xml' % self.workdir,
'-publish', resolver,
'-publishpattern', '%s/[organisation]/[module]/'
'[artifact]-[revision](-[classifier]).[ext]' % self.workdir,
'-revision', entry.version().version(),
'-m2compatible',
]
# TODO(John Sirois): global logging options should be hidden behind some sort of log manager
# that we can:
# a.) obtain a handle to (dependency injection or manual plumbing)
# b.) query for log detail, ie: `if log_manager.is_verbose:`
if self.get_options().level == 'debug':
args.append('-verbose')
if self.local_snapshot:
args.append('-overwrite')
try:
ivy.execute(jvm_options=jvm_options, args=args,
workunit_factory=self.context.new_workunit, workunit_name='jar-publish')
except Ivy.Error as e:
raise TaskError('Failed to push {0}! {1}'.format(pushdb_coordinate(jar, entry), e))
def _do_resolve(self, confs, executor, extra_args, global_vts, pinned_artifacts,
raw_target_classpath_file, resolve_hash_name, resolve_workdir,
workunit_name):
safe_mkdir(resolve_workdir)
ivy = Bootstrapper.default_ivy(bootstrap_workunit_factory=self.context.new_workunit)
with safe_concurrent_creation(raw_target_classpath_file) as raw_target_classpath_file_tmp:
args = ['-cachepath', raw_target_classpath_file_tmp] + extra_args
targets = global_vts.targets
# TODO(John Sirois): merge the code below into IvyUtils or up here; either way, better
# diagnostics can be had in `IvyUtils.generate_ivy` if this is done.
# See: https://github.com/pantsbuild/pants/issues/2239
jars, global_excludes = IvyUtils.calculate_classpath(targets)
# Don't pass global excludes to ivy when using soft excludes.
if self.get_options().soft_excludes:
global_excludes = []
ivyxml = self._ivy_xml_path(resolve_workdir)
with IvyUtils.ivy_lock:
try:
IvyUtils.generate_ivy(targets, jars, global_excludes, ivyxml, confs,
resolve_hash_name, pinned_artifacts)
except IvyUtils.IvyError as e:
raise self.Error('Failed to prepare ivy resolve: {}'.format(e))
self._exec_ivy(ivy, executor, confs, ivyxml, args, workunit_name)
# Copy ivy resolve file into resolve workdir.
for conf in confs:
atomic_copy(IvyUtils.xml_report_path(self.ivy_cache_dir, resolve_hash_name, conf),
self._resolve_report_path(resolve_workdir, conf))
if not os.path.exists(raw_target_classpath_file_tmp):
raise self.Error('Ivy failed to create classpath file at {}'
.format(raw_target_classpath_file_tmp))
logger.debug('Moved ivy classfile file to {dest}'.format(dest=raw_target_classpath_file))
def mapjars(self, genmap, target, executor, workunit_factory=None):
"""
Parameters:
genmap: the jar_dependencies ProductMapping entry for the required products.
target: the target whose jar dependencies are being retrieved.
"""
mapdir = os.path.join(self.mapto_dir(), target.id)
safe_mkdir(mapdir, clean=True)
ivyargs = [
'-retrieve', '%s/[organisation]/[artifact]/[conf]/'
'[organisation]-[artifact]-[revision](-[classifier]).[ext]' % mapdir,
'-symlink',
]
self.exec_ivy(mapdir,
[target],
ivyargs,
confs=target.payload.configurations,
ivy=Bootstrapper.default_ivy(executor),
workunit_factory=workunit_factory,
workunit_name='map-jars')
for org in os.listdir(mapdir):
orgdir = os.path.join(mapdir, org)
if os.path.isdir(orgdir):
for name in os.listdir(orgdir):
artifactdir = os.path.join(orgdir, name)
if os.path.isdir(artifactdir):
for conf in os.listdir(artifactdir):
confdir = os.path.join(artifactdir, conf)
for f in os.listdir(confdir):
if self.is_mappable_artifact(org, name, f):
# TODO(John Sirois): kill the org and (org, name) exclude mappings in favor of a
# conf whitelist
genmap.add(org, confdir).append(f)
genmap.add((org, name), confdir).append(f)
genmap.add(target, confdir).append(f)
genmap.add((target, conf), confdir).append(f)
genmap.add((org, name, conf), confdir).append(f)
def test_default_ivy(self):
with subsystem_instance(IvySubsystem):
ivy = Bootstrapper.default_ivy()
self.assertIsNotNone(ivy.ivy_cache_dir)
self.assertIsNone(ivy.ivy_settings)
def mapjars(self, genmap, target, executor, jars=None):
"""Resolves jars for the target and stores their locations in genmap.
:param genmap: The jar_dependencies ProductMapping entry for the required products.
:param target: The target whose jar dependencies are being retrieved.
:param jars: If specified, resolves the given jars rather than
:type jars: List of :class:`pants.backend.jvm.targets.jar_dependency.JarDependency` (jar())
objects.
Here is an example of what the resulting genmap looks like after sucessfully mapping
a JarLibrary target with a single JarDependency:
ProductMapping(ivy_imports) {
# target
UnpackedJars(BuildFileAddress(.../unpack/BUILD, foo)) =>
.../.pants.d/test/IvyImports/mapped-jars/unpack.foo/com.example/bar/default
[u'com.example-bar-0.0.1.jar']
# (org, name)
(u'com.example', u'bar') =>
.../.pants.d/test/IvyImports/mapped-jars/unpack.foo/com.example/bar/default
[u'com.example-bar-0.0.1.jar']
# (org)
com.example => .../.pants.d/test/IvyImports/mapped-jars/unpack.foo/com.example/bar/default
[u'com.example-bar-0.0.1.jar']
# (target)
(UnpackedJars(BuildFileAddress(.../unpack/BUILD, foo)), u'default') =>
.../.pants.d/test/IvyImports/mapped-jars/unpack.foo/com.example/bar/default
[u'com.example-bar-0.0.1.jar']
# (org, name, conf)
(u'com.example', u'bar', u'default') => .../.pants.d/test/IvyImports/mapped-jars/unpack.foo/com.example/bar/default
[u'com.example-bar-0.0.1.jar']
"""
mapdir = self.mapjar_workdir(target)
safe_mkdir(mapdir, clean=True)
ivyargs = [
'-retrieve', '{}/[organisation]/[artifact]/[conf]/'
'[organisation]-[artifact]-[revision](-[classifier]).[ext]'.format(mapdir),
'-symlink',
]
confs = maybe_list(target.payload.get_field_value('configurations') or [])
self.exec_ivy(mapdir,
[target],
executor=executor,
args=ivyargs,
confs=confs,
ivy=Bootstrapper.default_ivy(),
workunit_name='map-jars',
jars=jars,
use_soft_excludes=False)
for org in os.listdir(mapdir):
orgdir = os.path.join(mapdir, org)
if os.path.isdir(orgdir):
for name in os.listdir(orgdir):
artifactdir = os.path.join(orgdir, name)
if os.path.isdir(artifactdir):
for conf in os.listdir(artifactdir):
confdir = os.path.join(artifactdir, conf)
for f in os.listdir(confdir):
# TODO(John Sirois): kill the org and (org, name) exclude mappings in favor of a
# conf whitelist
genmap.add(org, confdir).append(f)
genmap.add((org, name), confdir).append(f)
genmap.add(target, confdir).append(f)
genmap.add((target, conf), confdir).append(f)
genmap.add((org, name, conf), confdir).append(f)
def ivy_resolve(self,
targets,
executor=None,
silent=False,
workunit_name=None,
confs=None,
custom_args=None):
"""Executes an ivy resolve for the relevant subset of the given targets.
Returns the resulting classpath, and the set of relevant targets. Also populates
the 'ivy_resolve_symlink_map' product for jars resulting from the resolve."""
if not targets:
return ([], set())
# NOTE: Always pass all the targets to exec_ivy, as they're used to calculate the name of
# the generated module, which in turn determines the location of the XML report file
# ivy generates. We recompute this name from targets later in order to find that file.
# TODO: This is fragile. Refactor so that we're not computing the name twice.
ivy = Bootstrapper.default_ivy(bootstrap_workunit_factory=self.context.new_workunit)
ivy_workdir = os.path.join(self.context.options.for_global_scope().pants_workdir, 'ivy')
fingerprint_strategy = IvyResolveFingerprintStrategy(confs)
with self.invalidated(targets,
invalidate_dependents=False,
silent=silent,
fingerprint_strategy=fingerprint_strategy) as invalidation_check:
if not invalidation_check.all_vts:
return ([], set())
global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)
# If a report file is not present, we need to exec ivy, even if all the individual
# targets up to date... See https://rbcommons.com/s/twitter/r/2015
report_missing = False
report_confs = confs or ['default']
report_paths = []
for conf in report_confs:
report_path = IvyUtils.xml_report_path(global_vts.targets, conf)
if not os.path.exists(report_path):
report_missing = True
break
else:
report_paths.append(report_path)
target_workdir = os.path.join(ivy_workdir, global_vts.cache_key.hash)
target_classpath_file = os.path.join(target_workdir, 'classpath')
raw_target_classpath_file = target_classpath_file + '.raw'
raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp'
# A common dir for symlinks into the ivy2 cache. This ensures that paths to jars
# in artifact-cached analysis files are consistent across systems.
# Note that we have one global, well-known symlink dir, again so that paths are
# consistent across builds.
symlink_dir = os.path.join(ivy_workdir, 'jars')
# Note that it's possible for all targets to be valid but for no classpath file to exist at
# target_classpath_file, e.g., if we previously built a superset of targets.
if report_missing or invalidation_check.invalid_vts or not os.path.exists(raw_target_classpath_file):
args = ['-cachepath', raw_target_classpath_file_tmp] + (custom_args if custom_args else [])
self.exec_ivy(
target_workdir=target_workdir,
targets=global_vts.targets,
args=args,
executor=executor,
ivy=ivy,
workunit_name=workunit_name,
confs=confs,
use_soft_excludes=self.get_options().soft_excludes)
if not os.path.exists(raw_target_classpath_file_tmp):
raise TaskError('Ivy failed to create classpath file at {}'
.format(raw_target_classpath_file_tmp))
shutil.move(raw_target_classpath_file_tmp, raw_target_classpath_file)
logger.debug('Moved ivy classfile file to {dest}'.format(dest=raw_target_classpath_file))
if self.artifact_cache_writes_enabled():
self.update_artifact_cache([(global_vts, [raw_target_classpath_file])])
else:
logger.debug("Using previously resolved reports: {}".format(report_paths))
# Make our actual classpath be symlinks, so that the paths are uniform across systems.
# Note that we must do this even if we read the raw_target_classpath_file from the artifact
# cache. If we cache the target_classpath_file we won't know how to create the symlinks.
with IvyTaskMixin.symlink_map_lock:
products = self.context.products
existing_symlinks_map = products.get_data('ivy_resolve_symlink_map', lambda: dict())
symlink_map = IvyUtils.symlink_cachepath(
IvySubsystem.global_instance().get_options().cache_dir,
raw_target_classpath_file,
symlink_dir,
target_classpath_file,
existing_symlinks_map)
existing_symlinks_map.update(symlink_map)
with IvyUtils.cachepath(target_classpath_file) as classpath:
stripped_classpath = [path.strip() for path in classpath]
return (stripped_classpath, global_vts.targets)
def ivy_resolve(self,
targets,
executor=None,
silent=False,
workunit_name=None,
confs=None,
custom_args=None):
"""Executes an ivy resolve for the relevant subset of the given targets.
:returns: the resulting classpath, and the unique part of the name used for the resolution
report (a hash). Also populates the 'ivy_resolve_symlink_map' product for jars resulting
from the resolve."""
if not targets:
return ([], None)
ivy = Bootstrapper.default_ivy(bootstrap_workunit_factory=self.context.new_workunit)
ivy_workdir = os.path.join(self.context.options.for_global_scope().pants_workdir, 'ivy')
fingerprint_strategy = IvyResolveFingerprintStrategy(confs)
with self.invalidated(targets,
invalidate_dependents=False,
silent=silent,
fingerprint_strategy=fingerprint_strategy) as invalidation_check:
if not invalidation_check.all_vts:
return ([], None)
global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)
# If a report file is not present, we need to exec ivy, even if all the individual
# targets up to date... See https://rbcommons.com/s/twitter/r/2015
report_missing = False
report_confs = confs or ['default']
report_paths = []
resolve_hash_name = global_vts.cache_key.hash
for conf in report_confs:
report_path = IvyUtils.xml_report_path(resolve_hash_name, conf)
if not os.path.exists(report_path):
report_missing = True
break
else:
report_paths.append(report_path)
target_workdir = os.path.join(ivy_workdir, resolve_hash_name)
target_classpath_file = os.path.join(target_workdir, 'classpath')
raw_target_classpath_file = target_classpath_file + '.raw'
raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp'
# A common dir for symlinks into the ivy2 cache. This ensures that paths to jars
# in artifact-cached analysis files are consistent across systems.
# Note that we have one global, well-known symlink dir, again so that paths are
# consistent across builds.
symlink_dir = os.path.join(ivy_workdir, 'jars')
# Note that it's possible for all targets to be valid but for no classpath file to exist at
# target_classpath_file, e.g., if we previously built a superset of targets.
if report_missing or invalidation_check.invalid_vts or not os.path.exists(raw_target_classpath_file):
args = ['-cachepath', raw_target_classpath_file_tmp] + (custom_args if custom_args else [])
self.exec_ivy(
target_workdir=target_workdir,
targets=global_vts.targets,
args=args,
executor=executor,
ivy=ivy,
workunit_name=workunit_name,
confs=confs,
use_soft_excludes=self.get_options().soft_excludes,
resolve_hash_name=resolve_hash_name)
if not os.path.exists(raw_target_classpath_file_tmp):
raise TaskError('Ivy failed to create classpath file at {}'
.format(raw_target_classpath_file_tmp))
shutil.move(raw_target_classpath_file_tmp, raw_target_classpath_file)
logger.debug('Moved ivy classfile file to {dest}'.format(dest=raw_target_classpath_file))
if self.artifact_cache_writes_enabled():
self.update_artifact_cache([(global_vts, [raw_target_classpath_file])])
else:
logger.debug("Using previously resolved reports: {}".format(report_paths))
# Make our actual classpath be symlinks, so that the paths are uniform across systems.
# Note that we must do this even if we read the raw_target_classpath_file from the artifact
# cache. If we cache the target_classpath_file we won't know how to create the symlinks.
with IvyTaskMixin.symlink_map_lock:
products = self.context.products
existing_symlinks_map = products.get_data('ivy_resolve_symlink_map', lambda: dict())
symlink_map = IvyUtils.symlink_cachepath(
IvySubsystem.global_instance().get_options().cache_dir,
raw_target_classpath_file,
symlink_dir,
target_classpath_file,
existing_symlinks_map)
existing_symlinks_map.update(symlink_map)
with IvyUtils.cachepath(target_classpath_file) as classpath:
stripped_classpath = [path.strip() for path in classpath]
return (stripped_classpath, resolve_hash_name)
def _ivy_resolve(self,
targets,
executor=None,
silent=False,
workunit_name=None,
confs=None,
extra_args=None,
invalidate_dependents=False,
pinned_artifacts=None):
"""Resolves external dependencies for the given targets.
If there are no targets suitable for jvm transitive dependency resolution, an empty result is
returned, ie: ([], {}, None).
:param targets: The targets to resolve jvm dependencies for.
:type targets: :class:`collections.Iterable` of :class:`pants.build_graph.target.Target`
:param executor: A java executor to run ivy with.
:type executor: :class:`pants.java.executor.Executor`
:param confs: The ivy configurations to resolve; ('default',) by default.
:type confs: :class:`collections.Iterable` of string
:param extra_args: Any extra command line arguments to pass to ivy.
:type extra_args: list of string
:param bool invalidate_dependents: `True` to invalidate dependents of targets that needed to be
resolved.
:returns: A tuple of the classpath, a mapping from ivy cache jars to their linked location
under .pants.d, and the id of the reports associated with the resolve.
:rtype: tuple of (list, dict, string)
"""
if not targets:
return [], {}, None
confs = confs or ('default',)
extra_args = extra_args or []
fingerprint_strategy = IvyResolveFingerprintStrategy(confs)
# NB: See class pydoc regarding `use_cache=False`.
with self.invalidated(targets,
invalidate_dependents=invalidate_dependents,
silent=silent,
fingerprint_strategy=fingerprint_strategy,
use_cache=False) as invalidation_check:
if not invalidation_check.all_vts:
return [], {}, None
global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)
resolve_hash_name = global_vts.cache_key.hash
ivy_workdir = os.path.join(self.context.options.for_global_scope().pants_workdir, 'ivy')
target_workdir = os.path.join(ivy_workdir, resolve_hash_name)
target_classpath_file = os.path.join(target_workdir, 'classpath')
raw_target_classpath_file = target_classpath_file + '.raw'
# If a report file is not present, we need to exec ivy, even if all the individual
# targets are up to date. See https://rbcommons.com/s/twitter/r/2015.
# Note that it's possible for all targets to be valid but for no classpath file to exist at
# target_classpath_file, e.g., if we previously built a superset of targets.
any_report_missing, existing_report_paths = self._collect_existing_reports(confs, resolve_hash_name)
if (invalidation_check.invalid_vts or
any_report_missing or
not os.path.exists(raw_target_classpath_file)):
ivy = Bootstrapper.default_ivy(bootstrap_workunit_factory=self.context.new_workunit)
raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp'
args = ['-cachepath', raw_target_classpath_file_tmp] + extra_args
self._exec_ivy(
target_workdir=target_workdir,
targets=global_vts.targets,
args=args,
executor=executor,
ivy=ivy,
workunit_name=workunit_name,
confs=confs,
use_soft_excludes=self.get_options().soft_excludes,
resolve_hash_name=resolve_hash_name,
pinned_artifacts=pinned_artifacts)
if not os.path.exists(raw_target_classpath_file_tmp):
raise self.Error('Ivy failed to create classpath file at {}'
.format(raw_target_classpath_file_tmp))
shutil.move(raw_target_classpath_file_tmp, raw_target_classpath_file)
logger.debug('Moved ivy classfile file to {dest}'.format(dest=raw_target_classpath_file))
else:
logger.debug("Using previously resolved reports: {}".format(existing_report_paths))
# Make our actual classpath be symlinks, so that the paths are uniform across systems.
# Note that we must do this even if we read the raw_target_classpath_file from the artifact
# cache. If we cache the target_classpath_file we won't know how to create the symlinks.
with IvyTaskMixin.symlink_map_lock:
# A common dir for symlinks into the ivy2 cache. This ensures that paths to jars
# in artifact-cached analysis files are consistent across systems.
# Note that we have one global, well-known symlink dir, again so that paths are
# consistent across builds.
symlink_dir = os.path.join(ivy_workdir, 'jars')
symlink_map = IvyUtils.symlink_cachepath(self.ivy_cache_dir,
raw_target_classpath_file,
symlink_dir,
target_classpath_file)
classpath = IvyUtils.load_classpath_from_cachepath(target_classpath_file)
return classpath, symlink_map, resolve_hash_name
def mapjars(self, genmap, target, executor, jars=None):
"""Resolves jars for the target and stores their locations in genmap.
:param genmap: The jar_dependencies ProductMapping entry for the required products.
:param target: The target whose jar dependencies are being retrieved.
:param jars: If specified, resolves the given jars rather than
:type jars: List of :class:`pants.backend.jvm.targets.jar_dependency.JarDependency` (jar())
objects.
Here is an example of what the resulting genmap looks like after sucessfully mapping
a JarLibrary target with a single JarDependency:
ProductMapping(ivy_imports) {
# target
UnpackedJars(BuildFileAddress(.../unpack/BUILD, foo)) =>
.../.pants.d/test/IvyImports/mapped-jars/unpack.foo/com.example/bar/default
[u'com.example-bar-0.0.1.jar']
# (org, name)
(u'com.example', u'bar') =>
.../.pants.d/test/IvyImports/mapped-jars/unpack.foo/com.example/bar/default
[u'com.example-bar-0.0.1.jar']
# (org)
com.example => .../.pants.d/test/IvyImports/mapped-jars/unpack.foo/com.example/bar/default
[u'com.example-bar-0.0.1.jar']
# (target)
(UnpackedJars(BuildFileAddress(.../unpack/BUILD, foo)), u'default') =>
.../.pants.d/test/IvyImports/mapped-jars/unpack.foo/com.example/bar/default
[u'com.example-bar-0.0.1.jar']
# (org, name, conf)
(u'com.example', u'bar', u'default') => .../.pants.d/test/IvyImports/mapped-jars/unpack.foo/com.example/bar/default
[u'com.example-bar-0.0.1.jar']
"""
mapdir = self.mapjar_workdir(target)
safe_mkdir(mapdir, clean=True)
ivyargs = self._get_ivy_args(mapdir)
confs = maybe_list(target.payload.get_field_value('configurations') or [])
self.exec_ivy(mapdir,
[target],
executor=executor,
args=ivyargs,
confs=confs,
ivy=Bootstrapper.default_ivy(),
workunit_name='map-jars',
jars=jars,
use_soft_excludes=False)
for org in os.listdir(mapdir):
orgdir = os.path.join(mapdir, org)
if os.path.isdir(orgdir):
for name in os.listdir(orgdir):
artifactdir = os.path.join(orgdir, name)
if os.path.isdir(artifactdir):
for conf in os.listdir(artifactdir):
confdir = os.path.join(artifactdir, conf)
for f in os.listdir(confdir):
# TODO(John Sirois): kill the org and (org, name) exclude mappings in favor of a
# conf whitelist
genmap.add(org, confdir).append(f)
genmap.add((org, name), confdir).append(f)
genmap.add(target, confdir).append(f)
genmap.add((target, conf), confdir).append(f)
genmap.add((org, name, conf), confdir).append(f)
def publish(self, publications, jar, version, repo, published):
"""Run ivy to publish a jar.
:param str ivyxml_path: The path to the ivy file.
:param list published: A list of jars published so far (including this one).
"""
try:
ivy = Bootstrapper.default_ivy()
except Bootstrapper.Error as e:
raise TaskError('Failed to push {0}! {1}'.format(jar, e))
if ivy.ivy_settings is None:
raise TaskError(
'An ivysettings.xml with writeable resolvers is required for publishing, '
'but none was configured.')
path = repo.get('path')
ivysettings = self.generate_ivysettings(self.fetch_ivysettings(ivy),
published,
publish_local=path)
ivyxml = self.generate_ivy(jar, version, publications)
resolver = repo['resolver']
args = [
'-settings',
ivysettings,
'-ivy',
ivyxml,
# Without this setting, the ivy.xml is delivered to the CWD, littering the workspace. We
# don't need the ivy.xml, so just give it path under the workdir we won't use.
'-deliverto',
ivyxml + '.unused',
'-publish',
resolver,
'-publishpattern',
'{}/[organisation]/[module]/'
'[artifact]-[revision](-[classifier]).[ext]'.format(self.workdir),
'-revision',
version,
'-m2compatible',
]
if self.get_options().level == 'debug':
args.append('-verbose')
if self.get_options().local:
args.append('-overwrite')
if not self.get_options().dryrun:
try:
ivy.execute(jvm_options=self._ivy_jvm_options(repo),
args=args,
workunit_factory=self.context.new_workunit,
workunit_name='ivy-publish')
except Ivy.Error as e:
raise TaskError('Failed to push {0}! {1}'.format(jar, e))
else:
print("\nDRYRUN- would have pushed:{} using {} resolver.".format(
self.jar_coordinate(jar, version), resolver))
def test_default_ivy(self): ivy = Bootstrapper.default_ivy() self.assertIsNotNone(ivy.ivy_cache_dir) self.assertIsNotNone(ivy.ivy_settings)
def ivy_resolve(self,
targets,
executor=None,
symlink_ivyxml=False,
silent=False,
workunit_name=None,
workunit_labels=None):
# NOTE: Always pass all the targets to exec_ivy, as they're used to calculate the name of
# the generated module, which in turn determines the location of the XML report file
# ivy generates. We recompute this name from targets later in order to find that file.
# TODO: This is fragile. Refactor so that we're not computing the name twice.
if executor and not isinstance(executor, Executor):
raise ValueError('The executor must be an Executor instance, given %s of type %s'
% (executor, type(executor)))
ivy = Bootstrapper.default_ivy(java_executor=executor,
bootstrap_workunit_factory=self.context.new_workunit)
if not targets:
return []
ivy_workdir = os.path.join(self.context.config.getdefault('pants_workdir'), 'ivy')
ivy_utils = IvyUtils(config=self.context.config,
options=self.context.options,
log=self.context.log)
fingerprint_strategy = IvyResolveFingerprintStrategy()
with self.invalidated(targets,
invalidate_dependents=True,
silent=silent,
fingerprint_strategy=fingerprint_strategy) as invalidation_check:
global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)
target_workdir = os.path.join(ivy_workdir, global_vts.cache_key.hash)
target_classpath_file = os.path.join(target_workdir, 'classpath')
raw_target_classpath_file = target_classpath_file + '.raw'
raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp'
# A common dir for symlinks into the ivy2 cache. This ensures that paths to jars
# in artifact-cached analysis files are consistent across systems.
# Note that we have one global, well-known symlink dir, again so that paths are
# consistent across builds.
symlink_dir = os.path.join(ivy_workdir, 'jars')
# Note that it's possible for all targets to be valid but for no classpath file to exist at
# target_classpath_file, e.g., if we previously built a superset of targets.
if invalidation_check.invalid_vts or not os.path.exists(raw_target_classpath_file):
args = ['-cachepath', raw_target_classpath_file_tmp]
def exec_ivy():
ivy_utils.exec_ivy(
target_workdir=target_workdir,
targets=targets,
args=args,
ivy=ivy,
workunit_name='ivy',
workunit_factory=self.context.new_workunit,
symlink_ivyxml=symlink_ivyxml)
if workunit_name:
with self.context.new_workunit(name=workunit_name, labels=workunit_labels or []):
exec_ivy()
else:
exec_ivy()
if not os.path.exists(raw_target_classpath_file_tmp):
raise TaskError('Ivy failed to create classpath file at %s'
% raw_target_classpath_file_tmp)
shutil.move(raw_target_classpath_file_tmp, raw_target_classpath_file)
if self.artifact_cache_writes_enabled():
self.update_artifact_cache([(global_vts, [raw_target_classpath_file])])
# Make our actual classpath be symlinks, so that the paths are uniform across systems.
# Note that we must do this even if we read the raw_target_classpath_file from the artifact
# cache. If we cache the target_classpath_file we won't know how to create the symlinks.
symlink_map = IvyUtils.symlink_cachepath(self.context.ivy_home, raw_target_classpath_file,
symlink_dir, target_classpath_file)
with IvyTaskMixin.symlink_map_lock:
all_symlinks_map = self.context.products.get_data('symlink_map') or defaultdict(list)
for path, symlink in symlink_map.items():
all_symlinks_map[os.path.realpath(path)].append(symlink)
self.context.products.safe_create_data('symlink_map', lambda: all_symlinks_map)
with IvyUtils.cachepath(target_classpath_file) as classpath:
stripped_classpath = [path.strip() for path in classpath]
return [path for path in stripped_classpath if ivy_utils.is_classpath_artifact(path)]
def ivy_resolve(self,
targets,
executor=None,
silent=False,
workunit_name=None,
confs=None,
custom_args=None):
"""Executes an ivy resolve for the relevant subset of the given targets.
Returns the resulting classpath, and the set of relevant targets. Also populates
the 'ivy_resolve_symlink_map' product for jars resulting from the resolve."""
if not targets:
return ([], set())
# NOTE: Always pass all the targets to exec_ivy, as they're used to calculate the name of
# the generated module, which in turn determines the location of the XML report file
# ivy generates. We recompute this name from targets later in order to find that file.
# TODO: This is fragile. Refactor so that we're not computing the name twice.
ivy = Bootstrapper.default_ivy(
bootstrap_workunit_factory=self.context.new_workunit)
ivy_workdir = os.path.join(
self.context.options.for_global_scope().pants_workdir, 'ivy')
fingerprint_strategy = IvyResolveFingerprintStrategy()
with self.invalidated(targets,
invalidate_dependents=False,
silent=silent,
fingerprint_strategy=fingerprint_strategy
) as invalidation_check:
if not invalidation_check.all_vts:
return ([], set())
global_vts = VersionedTargetSet.from_versioned_targets(
invalidation_check.all_vts)
target_workdir = os.path.join(ivy_workdir,
global_vts.cache_key.hash)
target_classpath_file = os.path.join(target_workdir, 'classpath')
raw_target_classpath_file = target_classpath_file + '.raw'
raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp'
# A common dir for symlinks into the ivy2 cache. This ensures that paths to jars
# in artifact-cached analysis files are consistent across systems.
# Note that we have one global, well-known symlink dir, again so that paths are
# consistent across builds.
symlink_dir = os.path.join(ivy_workdir, 'jars')
# Note that it's possible for all targets to be valid but for no classpath file to exist at
# target_classpath_file, e.g., if we previously built a superset of targets.
if invalidation_check.invalid_vts or not os.path.exists(
raw_target_classpath_file):
args = ['-cachepath', raw_target_classpath_file_tmp
] + (custom_args if custom_args else [])
self.exec_ivy(target_workdir=target_workdir,
targets=global_vts.targets,
args=args,
executor=executor,
ivy=ivy,
workunit_name=workunit_name,
confs=confs)
if not os.path.exists(raw_target_classpath_file_tmp):
raise TaskError(
'Ivy failed to create classpath file at %s' %
raw_target_classpath_file_tmp)
shutil.move(raw_target_classpath_file_tmp,
raw_target_classpath_file)
logger.debug('Moved ivy classfile file to {dest}'.format(
dest=raw_target_classpath_file))
if self.artifact_cache_writes_enabled():
self.update_artifact_cache([(global_vts,
[raw_target_classpath_file])])
# Make our actual classpath be symlinks, so that the paths are uniform across systems.
# Note that we must do this even if we read the raw_target_classpath_file from the artifact
# cache. If we cache the target_classpath_file we won't know how to create the symlinks.
with IvyTaskMixin.symlink_map_lock:
products = self.context.products
existing_symlinks_map = products.get_data(
'ivy_resolve_symlink_map', lambda: dict())
symlink_map = IvyUtils.symlink_cachepath(
ivy.ivy_cache_dir, raw_target_classpath_file, symlink_dir,
target_classpath_file, existing_symlinks_map)
existing_symlinks_map.update(symlink_map)
with IvyUtils.cachepath(target_classpath_file) as classpath:
stripped_classpath = [path.strip() for path in classpath]
return (stripped_classpath, global_vts.targets)
def publish(self, publications, jar, entry, repo, published):
"""Run ivy to publish a jar.
ivyxml_path is the path to the ivy file; published is a list of jars published so far
(including this one). entry is a pushdb entry.
"""
try:
ivy = Bootstrapper.default_ivy()
except Bootstrapper.Error as e:
raise TaskError(
f"Failed to push {pushdb_coordinate(jar, entry)}! {e!r}")
path = repo.get("path")
ivysettings = self.generate_ivysettings(ivy,
published,
publish_local=path)
version = entry.version().version()
ivyxml = self.generate_ivy(jar, version, publications)
resolver = repo["resolver"]
args = [
"-settings",
ivysettings,
"-ivy",
ivyxml,
# Without this setting, the ivy.xml is delivered to the CWD, littering the workspace. We
# don't need the ivy.xml, so just give it path under the workdir we won't use.
"-deliverto",
ivyxml + ".unused",
"-publish",
resolver,
"-publishpattern",
"{}/[organisation]/[module]/"
"[artifact]-[revision](-[classifier]).[ext]".format(self.workdir),
"-revision",
version,
"-m2compatible",
]
# TODO(John Sirois): global logging options should be hidden behind some sort of log manager
# that we can:
# a.) obtain a handle to (dependency injection or manual plumbing)
# b.) query for log detail, ie: `if log_manager.is_verbose:`
if self.debug:
args.append("-verbose")
if self.local_snapshot:
args.append("-overwrite")
try:
jvm_options = self._ivy_jvm_options(repo)
ivy.execute(
jvm_options=jvm_options,
args=args,
workunit_factory=self.context.new_workunit,
workunit_name="ivy-publish",
)
except Ivy.Error as e:
raise TaskError(
f"Failed to push {pushdb_coordinate(jar, entry)}! {e!r}")
def ivy_resolve(self,
targets,
executor=None,
silent=False,
workunit_name=None,
confs=None,
custom_args=None,
invalidate_dependents=False):
"""Resolves external dependencies for the given targets.
If there are no targets suitable for jvm transitive dependency resolution, an empty result is
returned, ie: ([], {}, None).
:param targets: The targets to resolve jvm dependencies for.
:type targets: :class:`collections.Iterable` of :class:`pants.build_graph.target.Target`
:param executor: A java executor to run ivy with.
:type executor: :class:`pants.java.executor.Executor`
:param confs: The ivy configurations to resolve; ('default',) by default.
:type confs: :class:`collections.Iterable` of string
:param custom_args: Any extra command line arguments to pass to ivy.
:type custom_args: list of string
:param bool invalidate_dependents: `True` to invalidate dependents of targets that needed to be
resolved.
:returns: A tuple of the classpath, a mapping from ivy cache jars to their linked location
under .pants.d, and the id of the reports associated with the resolve.
:rtype: tuple of (list, dict, string)
"""
if not targets:
return [], {}, None
ivy = Bootstrapper.default_ivy(
bootstrap_workunit_factory=self.context.new_workunit)
ivy_workdir = os.path.join(
self.context.options.for_global_scope().pants_workdir, 'ivy')
fingerprint_strategy = IvyResolveFingerprintStrategy(confs)
with self.invalidated(targets,
invalidate_dependents=invalidate_dependents,
silent=silent,
fingerprint_strategy=fingerprint_strategy
) as invalidation_check:
if not invalidation_check.all_vts:
return [], {}, None
global_vts = VersionedTargetSet.from_versioned_targets(
invalidation_check.all_vts)
# If a report file is not present, we need to exec ivy, even if all the individual
# targets up to date... See https://rbcommons.com/s/twitter/r/2015
report_missing = False
report_confs = confs or ['default']
report_paths = []
resolve_hash_name = global_vts.cache_key.hash
for conf in report_confs:
report_path = IvyUtils.xml_report_path(self.ivy_cache_dir,
resolve_hash_name, conf)
if not os.path.exists(report_path):
report_missing = True
break
else:
report_paths.append(report_path)
target_workdir = os.path.join(ivy_workdir, resolve_hash_name)
target_classpath_file = os.path.join(target_workdir, 'classpath')
raw_target_classpath_file = target_classpath_file + '.raw'
raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp'
# A common dir for symlinks into the ivy2 cache. This ensures that paths to jars
# in artifact-cached analysis files are consistent across systems.
# Note that we have one global, well-known symlink dir, again so that paths are
# consistent across builds.
symlink_dir = os.path.join(ivy_workdir, 'jars')
# Note that it's possible for all targets to be valid but for no classpath file to exist at
# target_classpath_file, e.g., if we previously built a superset of targets.
if (report_missing or invalidation_check.invalid_vts
or not os.path.exists(raw_target_classpath_file)):
args = ['-cachepath', raw_target_classpath_file_tmp
] + (custom_args if custom_args else [])
self._exec_ivy(
target_workdir=target_workdir,
targets=global_vts.targets,
args=args,
executor=executor,
ivy=ivy,
workunit_name=workunit_name,
confs=confs,
use_soft_excludes=self.get_options().soft_excludes,
resolve_hash_name=resolve_hash_name)
if not os.path.exists(raw_target_classpath_file_tmp):
raise self.Error(
'Ivy failed to create classpath file at {}'.format(
raw_target_classpath_file_tmp))
shutil.move(raw_target_classpath_file_tmp,
raw_target_classpath_file)
logger.debug('Moved ivy classfile file to {dest}'.format(
dest=raw_target_classpath_file))
if self.artifact_cache_writes_enabled():
self.update_artifact_cache([(global_vts,
[raw_target_classpath_file])])
else:
logger.debug("Using previously resolved reports: {}".format(
report_paths))
# Make our actual classpath be symlinks, so that the paths are uniform across systems.
# Note that we must do this even if we read the raw_target_classpath_file from the artifact
# cache. If we cache the target_classpath_file we won't know how to create the symlinks.
with IvyTaskMixin.symlink_map_lock:
symlink_map = IvyUtils.symlink_cachepath(
self.ivy_cache_dir, raw_target_classpath_file, symlink_dir,
target_classpath_file)
with IvyUtils.cachepath(target_classpath_file) as classpath:
stripped_classpath = [path.strip() for path in classpath]
return stripped_classpath, symlink_map, resolve_hash_name
def test_junit_runner(self):
# Create the temporary base test directory
test_rel_path = 'tests/java/com/pants/foo'
test_abs_path = os.path.join(self.build_root, test_rel_path)
self.create_dir(test_rel_path)
# Generate the temporary java test source code.
test_java_file_rel_path = os.path.join(test_rel_path, 'FooTest.java')
test_java_file_abs_path = os.path.join(self.build_root,
test_java_file_rel_path)
self.create_file(
test_java_file_rel_path,
dedent('''
import org.junit.Test;
import static org.junit.Assert.assertTrue;
public class FooTest {
@Test
public void testFoo() {
assertTrue(5 > 3);
}
}
'''))
# Invoke ivy to resolve classpath for junit.
distribution = Distribution.cached(jdk=True)
executor = SubprocessExecutor(distribution=distribution)
classpath_file_abs_path = os.path.join(test_abs_path,
'junit.classpath')
ivy = Bootstrapper.default_ivy()
ivy.execute(args=[
'-cachepath', classpath_file_abs_path, '-dependency', 'junit',
'junit-dep', '4.10'
],
executor=executor)
with open(classpath_file_abs_path) as fp:
classpath = fp.read()
# Now directly invoking javac to compile the test java code into java class
# so later we can inject the class into products mapping for JUnitRun to execute
# the test on.
javac = distribution.binary('javac')
subprocess.check_call([
javac, '-d', test_abs_path, '-cp', classpath,
test_java_file_abs_path
])
# Create a java_tests target and a synthetic resource target.
java_tests = self.create_library(test_rel_path, 'java_tests',
'foo_test', ['FooTest.java'])
resources = self.make_target('some_resources', Resources)
# Set the context with the two targets, one java_tests target and
# one synthetic resources target.
# The synthetic resources target is to make sure we won't regress
# in the future with bug like https://github.com/pantsbuild/pants/issues/508. Note
# in that bug, the resources target must be the first one in the list.
context = self.context(target_roots=[resources, java_tests])
# Before we run the task, we need to inject the "classes_by_target" with
# the compiled test java classes that JUnitRun will know which test
# classes to execute. In a normal run, this "classes_by_target" will be
# populated by java compiling step.
class_products = context.products.get_data(
'classes_by_target', lambda: defaultdict(MultipleRootedProducts))
java_tests_products = MultipleRootedProducts()
java_tests_products.add_rel_paths(test_abs_path, ['FooTest.class'])
class_products[java_tests] = java_tests_products
# Also we need to add the FooTest.class's classpath to the compile_classpath
# products data mapping so JUnitRun will be able to add that into the final
# classpath under which the junit will be executed.
self.populate_compile_classpath(context=context,
classpath=[test_abs_path])
# Finally execute the task.
self.execute(context)
def ivy_resolve(self,
targets,
executor=None,
symlink_ivyxml=False,
silent=False,
workunit_name=None,
workunit_labels=None):
# NOTE: Always pass all the targets to exec_ivy, as they're used to calculate the name of
# the generated module, which in turn determines the location of the XML report file
# ivy generates. We recompute this name from targets later in order to find that file.
# TODO: This is fragile. Refactor so that we're not computing the name twice.
if executor and not isinstance(executor, Executor):
raise ValueError(
'The executor must be an Executor instance, given %s of type %s'
% (executor, type(executor)))
ivy = Bootstrapper.default_ivy(
java_executor=executor,
bootstrap_workunit_factory=self.context.new_workunit)
if not targets:
return []
ivy_workdir = os.path.join(
self.context.config.getdefault('pants_workdir'), 'ivy')
ivy_utils = IvyUtils(config=self.context.config,
options=self.context.options,
log=self.context.log)
fingerprint_strategy = IvyResolveFingerprintStrategy()
with self.invalidated(targets,
invalidate_dependents=True,
silent=silent,
fingerprint_strategy=fingerprint_strategy
) as invalidation_check:
global_vts = VersionedTargetSet.from_versioned_targets(
invalidation_check.all_vts)
target_workdir = os.path.join(ivy_workdir,
global_vts.cache_key.hash)
target_classpath_file = os.path.join(target_workdir, 'classpath')
raw_target_classpath_file = target_classpath_file + '.raw'
raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp'
# A common dir for symlinks into the ivy2 cache. This ensures that paths to jars
# in artifact-cached analysis files are consistent across systems.
# Note that we have one global, well-known symlink dir, again so that paths are
# consistent across builds.
symlink_dir = os.path.join(ivy_workdir, 'jars')
# Note that it's possible for all targets to be valid but for no classpath file to exist at
# target_classpath_file, e.g., if we previously built a superset of targets.
if invalidation_check.invalid_vts or not os.path.exists(
raw_target_classpath_file):
args = ['-cachepath', raw_target_classpath_file_tmp]
def exec_ivy():
ivy_utils.exec_ivy(
target_workdir=target_workdir,
targets=targets,
args=args,
ivy=ivy,
workunit_name='ivy',
workunit_factory=self.context.new_workunit,
symlink_ivyxml=symlink_ivyxml)
if workunit_name:
with self.context.new_workunit(name=workunit_name,
labels=workunit_labels
or []):
exec_ivy()
else:
exec_ivy()
if not os.path.exists(raw_target_classpath_file_tmp):
raise TaskError(
'Ivy failed to create classpath file at %s' %
raw_target_classpath_file_tmp)
shutil.move(raw_target_classpath_file_tmp,
raw_target_classpath_file)
logger.debug('Copied ivy classfile file to {dest}'.format(
dest=raw_target_classpath_file))
if self.artifact_cache_writes_enabled():
self.update_artifact_cache([(global_vts,
[raw_target_classpath_file])])
# Make our actual classpath be symlinks, so that the paths are uniform across systems.
# Note that we must do this even if we read the raw_target_classpath_file from the artifact
# cache. If we cache the target_classpath_file we won't know how to create the symlinks.
symlink_map = IvyUtils.symlink_cachepath(self.context.ivy_home,
raw_target_classpath_file,
symlink_dir,
target_classpath_file)
with IvyTaskMixin.symlink_map_lock:
all_symlinks_map = self.context.products.get_data(
'symlink_map') or defaultdict(list)
for path, symlink in symlink_map.items():
all_symlinks_map[os.path.realpath(path)].append(symlink)
self.context.products.safe_create_data('symlink_map',
lambda: all_symlinks_map)
with IvyUtils.cachepath(target_classpath_file) as classpath:
stripped_classpath = [path.strip() for path in classpath]
return [
path for path in stripped_classpath
if ivy_utils.is_classpath_artifact(path)
]
def ivy_resolve(self,
targets,
executor=None,
silent=False,
workunit_name=None,
confs=None,
custom_args=None):
"""Populates the product 'ivy_resolve_symlink_map' from the specified targets."""
if not targets:
return ([], set())
# NOTE: Always pass all the targets to exec_ivy, as they're used to calculate the name of
# the generated module, which in turn determines the location of the XML report file
# ivy generates. We recompute this name from targets later in order to find that file.
# TODO: This is fragile. Refactor so that we're not computing the name twice.
ivy = Bootstrapper.default_ivy(bootstrap_workunit_factory=self.context.new_workunit)
ivy_workdir = os.path.join(self.context.options.for_global_scope().pants_workdir, 'ivy')
fingerprint_strategy = IvyResolveFingerprintStrategy()
with self.invalidated(targets,
invalidate_dependents=False,
silent=silent,
fingerprint_strategy=fingerprint_strategy) as invalidation_check:
if not invalidation_check.all_vts:
return ([], set())
global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)
target_workdir = os.path.join(ivy_workdir, global_vts.cache_key.hash)
target_classpath_file = os.path.join(target_workdir, 'classpath')
raw_target_classpath_file = target_classpath_file + '.raw'
raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp'
# A common dir for symlinks into the ivy2 cache. This ensures that paths to jars
# in artifact-cached analysis files are consistent across systems.
# Note that we have one global, well-known symlink dir, again so that paths are
# consistent across builds.
symlink_dir = os.path.join(ivy_workdir, 'jars')
# Note that it's possible for all targets to be valid but for no classpath file to exist at
# target_classpath_file, e.g., if we previously built a superset of targets.
if invalidation_check.invalid_vts or not os.path.exists(raw_target_classpath_file):
args = ['-cachepath', raw_target_classpath_file_tmp] + (custom_args if custom_args else [])
self.exec_ivy(
target_workdir=target_workdir,
targets=global_vts.targets,
args=args,
executor=executor,
ivy=ivy,
workunit_name=workunit_name,
confs=confs)
if not os.path.exists(raw_target_classpath_file_tmp):
raise TaskError('Ivy failed to create classpath file at %s'
% raw_target_classpath_file_tmp)
shutil.move(raw_target_classpath_file_tmp, raw_target_classpath_file)
logger.debug('Copied ivy classfile file to {dest}'.format(dest=raw_target_classpath_file))
if self.artifact_cache_writes_enabled():
self.update_artifact_cache([(global_vts, [raw_target_classpath_file])])
# Make our actual classpath be symlinks, so that the paths are uniform across systems.
# Note that we must do this even if we read the raw_target_classpath_file from the artifact
# cache. If we cache the target_classpath_file we won't know how to create the symlinks.
symlink_map = IvyUtils.symlink_cachepath(ivy.ivy_cache_dir, raw_target_classpath_file,
symlink_dir, target_classpath_file)
with IvyTaskMixin.symlink_map_lock:
products = self.context.products
all_symlinks_map = products.get_data('ivy_resolve_symlink_map') or defaultdict(list)
for path, symlink in symlink_map.items():
all_symlinks_map[os.path.realpath(path)].append(symlink)
products.safe_create_data('ivy_resolve_symlink_map',
lambda: all_symlinks_map)
with IvyUtils.cachepath(target_classpath_file) as classpath:
stripped_classpath = [path.strip() for path in classpath]
return (stripped_classpath, global_vts.targets)
def test_junit_runner(self):
# Create the temporary base test directory
test_rel_path = 'tests/java/com/pants/foo'
test_abs_path = os.path.join(self.build_root, test_rel_path)
self.create_dir(test_rel_path)
# Generate the temporary java test source code.
test_java_file_rel_path = os.path.join(test_rel_path, 'FooTest.java')
test_java_file_abs_path = os.path.join(self.build_root, test_java_file_rel_path)
self.create_file(test_java_file_rel_path,
dedent('''
import org.junit.Test;
import static org.junit.Assert.assertTrue;
public class FooTest {
@Test
public void testFoo() {
assertTrue(5 > 3);
}
}
'''))
# Invoke ivy to resolve classpath for junit.
distribution = Distribution.cached(jdk=True)
executor = SubprocessExecutor(distribution=distribution)
classpath_file_abs_path = os.path.join(test_abs_path, 'junit.classpath')
ivy = Bootstrapper.default_ivy(java_executor=executor)
ivy.execute(args=['-cachepath', classpath_file_abs_path,
'-dependency', 'junit', 'junit-dep', '4.10'])
with open(classpath_file_abs_path) as fp:
classpath = fp.read()
# Now directly invoking javac to compile the test java code into java class
# so later we can inject the class into products mapping for JUnitRun to execute
# the test on.
javac = distribution.binary('javac')
subprocess.check_call(
[javac, '-d', test_abs_path, '-cp', classpath, test_java_file_abs_path])
# Create a java_tests target and a synthetic resource target.
java_tests = self.create_library(test_rel_path, 'java_tests', 'foo_test', ['FooTest.java'])
resources = self.make_target('some_resources', Resources)
# Set the context with the two targets, one java_tests target and
# one synthetic resources target.
# The synthetic resources target is to make sure we won't regress
# in the future with bug like https://github.com/pantsbuild/pants/issues/508. Note
# in that bug, the resources target must be the first one in the list.
context = self.context(target_roots=[resources, java_tests])
# Before we run the task, we need to inject the "classes_by_target" with
# the compiled test java classes that JUnitRun will know which test
# classes to execute. In a normal run, this "classes_by_target" will be
# populated by java compiling step.
class_products = context.products.get_data(
'classes_by_target', lambda: defaultdict(MultipleRootedProducts))
java_tests_products = MultipleRootedProducts()
java_tests_products.add_rel_paths(test_abs_path, ['FooTest.class'])
class_products[java_tests] = java_tests_products
# Also we need to add the FooTest.class's classpath to the exclusive_groups
# products data mapping so JUnitRun will be able to add that into the final
# classpath under which the junit will be executed.
self.populate_exclusive_groups(
context=context,
classpaths=[test_abs_path],
# This is a bit hacky. The issue in https://github.com/pantsbuild/pants/issues/508
# is that normal resources specified in the BUILD targets are fine, but the
# synthetic resources ones generated on the fly don't have exclusive_groups data
# mapping entries thus later on in _JUnitRunner lookup it blows up. So we manually
# exclude the synthetic resources target here to simulate that situation and ensure
# the _JUnitRunner does filter out all the non-java-tests targets.
target_predicate=lambda t: t != resources)
# Finally execute the task.
self.execute(context)
def test_default_ivy(self):
ivy = Bootstrapper.default_ivy()
self.assertIsNotNone(ivy.ivy_cache_dir)
self.assertIsNone(ivy.ivy_settings)
def test_default_ivy(self):
with subsystem_instance(IvySubsystem):
ivy = Bootstrapper.default_ivy()
self.assertIsNotNone(ivy.ivy_cache_dir)
self.assertIsNone(ivy.ivy_settings)
