def resolve_jars(self, targets):
# TODO: Why is this computed directly here instead of taking from the actual product
# computed by the {Ivy,Coursier}Resolve task?
executor = SubprocessExecutor(DistributionLocator.cached())
confs = []
if self.get_options().libraries:
confs.append('default')
if self.get_options().libraries_sources:
confs.append('sources')
if self.get_options().libraries_javadocs:
confs.append('javadoc')
compile_classpath = None
if confs:
compile_classpath = ClasspathProducts(
self.get_options().pants_workdir)
if JvmResolveSubsystem.global_instance().get_options(
).resolver == 'ivy':
IvyTaskMixin.resolve(self,
executor=executor,
targets=targets,
classpath_products=compile_classpath,
confs=confs)
else:
CoursierMixin.resolve(
self,
targets,
compile_classpath,
sources=self.get_options().libraries_sources,
javadoc=self.get_options().libraries_javadocs)
return compile_classpath
def execute(self,
jvm_options=None,
args=None,
executor=None,
workunit_factory=None,
workunit_name=None,
workunit_labels=None):
"""Executes the ivy commandline client with the given args.
Raises Ivy.Error if the command fails for any reason.
:param executor: Java executor to run ivy with.
"""
# NB(gmalmquist): It should be OK that we can't declare a subsystem_dependency in this file
# (because it's just a plain old object), because Ivy is only constructed by Bootstrapper, which
# makes an explicit call to IvySubsystem.global_instance() in its constructor, which in turn has
# a declared dependency on DistributionLocator.
executor = executor or SubprocessExecutor(DistributionLocator.cached())
runner = self.runner(jvm_options=jvm_options,
args=args,
executor=executor)
try:
with self.resolution_lock:
result = util.execute_runner(runner, workunit_factory,
workunit_name, workunit_labels)
if result != 0:
raise self.Error(
'Ivy command failed with exit code {}{}'.format(
result, ': ' + ' '.join(args) if args else ''))
except executor.Error as e:
raise self.Error('Problem executing ivy: {}'.format(e))
def create_scaladoc_command(self, classpath, gendir, *targets):
sources = []
for target in targets:
sources.extend(target.sources_relative_to_buildroot())
# TODO(Tejal Desai): pantsbuild/pants/65: Remove java_sources attribute for ScalaLibrary
# A '.scala' owning target may not have java_sources, eg: junit_tests
if hasattr(target, 'java_sources'):
for java_target in target.java_sources:
sources.extend(java_target.sources_relative_to_buildroot())
if not sources:
return None
scala_platform = ScalaPlatform.global_instance()
tool_classpath = scala_platform.compiler_classpath(
self.context.products)
args = ['-usejavacp', '-classpath', ':'.join(classpath), '-d', gendir]
args.extend(self.args)
args.extend(sources)
java_executor = SubprocessExecutor(DistributionLocator.cached())
runner = java_executor.runner(jvm_options=self.jvm_options,
classpath=tool_classpath,
main='scala.tools.nsc.ScalaDoc',
args=args)
return runner.command
def create_scaladoc_command(self, classpath, gendir, *targets):
sources = []
for target in targets:
sources.extend(target.sources_relative_to_buildroot())
# TODO(Tejal Desai): pantsbuild/pants/65: Remove java_sources attribute for ScalaLibrary
# A '.scala' owning target may not have java_sources, eg: junit_tests
if hasattr(target, "java_sources"):
for java_target in target.java_sources:
sources.extend(java_target.sources_relative_to_buildroot())
if not sources:
return None
scala_platform = ScalaPlatform.global_instance()
tool_classpath = [
cp_entry.path
for cp_entry in scala_platform.compiler_classpath_entries(self.context.products)
]
args = ["-usejavacp", "-classpath", ":".join(classpath), "-d", gendir]
args.extend(self.args)
args.extend(sources)
java_executor = SubprocessExecutor(self.preferred_jvm_distribution_for_targets(targets))
runner = java_executor.runner(
jvm_options=self.jvm_options,
classpath=tool_classpath,
main="scala.tools.nsc.ScalaDoc",
args=args,
)
return runner.command
def test_fails_with_bad_distribution(self):
class DefinitelyNotADistribution(object):
pass
with self.assertRaises(Executor.InvalidDistribution):
SubprocessExecutor(DefinitelyNotADistribution())
def execute_java(classpath, main, jvm_options=None, args=None, executor=None,
workunit_factory=None, workunit_name=None, workunit_labels=None,
cwd=None, workunit_log_config=None, distribution=None):
"""Executes the java program defined by the classpath and main.
If `workunit_factory` is supplied, does so in the context of a workunit.
:param list classpath: the classpath for the java program
:param string main: the fully qualified class name of the java program's entry point
:param list jvm_options: an optional sequence of options for the underlying jvm
:param list args: an optional sequence of args to pass to the java program
:param executor: an optional java executor to use to launch the program; defaults to a subprocess
spawn of the default java distribution
:param workunit_factory: an optional callable that can produce a workunit context
:param string workunit_name: an optional name for the work unit; defaults to the main
:param list workunit_labels: an optional sequence of labels for the work unit
:param string cwd: optionally set the working directory
:param WorkUnit.LogConfig workunit_log_config: an optional tuple of options affecting reporting
Returns the exit code of the java program.
Raises `pants.java.Executor.Error` if there was a problem launching java itself.
"""
executor = executor or SubprocessExecutor(distribution)
if not isinstance(executor, Executor):
raise ValueError('The executor argument must be a java Executor instance, give {} of type {}'
.format(executor, type(executor)))
runner = executor.runner(classpath, main, args=args, jvm_options=jvm_options, cwd=cwd)
workunit_name = workunit_name or main
return execute_runner(runner,
workunit_factory=workunit_factory,
workunit_name=workunit_name,
workunit_labels=workunit_labels,
cwd=cwd,
workunit_log_config=workunit_log_config)
def setUp(self):
self.jarjar = '/not/really/jarjar.jar'
with subsystem_instance(DistributionLocator):
executor = SubprocessExecutor(DistributionLocator.cached())
self.shader = Shader(jarjar_classpath=[self.jarjar],
executor=executor)
self.output_jar = '/not/really/shaded.jar'
def resolve_jars(self, targets):
executor = SubprocessExecutor(DistributionLocator.cached())
confs = []
if self.get_options().libraries:
confs.append('default')
if self.get_options().libraries_sources:
confs.append('sources')
if self.get_options().libraries_javadocs:
confs.append('javadoc')
compile_classpath = None
if confs:
compile_classpath = ClasspathProducts(
self.get_options().pants_workdir)
if JvmResolveSubsystem.global_instance().get_options(
).resolver == 'ivy':
IvyTaskMixin.resolve(self,
executor=executor,
targets=targets,
classpath_products=compile_classpath,
confs=confs)
else:
CoursierMixin.resolve(
self,
targets,
compile_classpath,
sources=self.get_options().libraries_sources,
javadoc=self.get_options().libraries_javadocs)
return compile_classpath
def resolve_jars(self, targets):
executor = SubprocessExecutor(DistributionLocator.cached())
confs = []
if self.get_options().libraries:
confs.append('default')
if self.get_options().libraries_sources:
confs.append('sources')
if self.get_options().libraries_javadocs:
confs.append('javadoc')
# TODO(gmalmquist): This is a terrible hack for backwards-compatibility with the pants-plugin.
# Kill it ASAP, and update test_export_integration#test_export_jar_path_with_excludes_soft to
# use the flag actually scoped for this task.
export_options = self.get_options()
ivy_options = self.context.options.for_scope('resolve.ivy')
for name in set.intersection(set(export_options), set(ivy_options)):
if not ivy_options.is_default(name):
setattr(export_options, name, ivy_options[name])
confs = confs or export_options.confs
compile_classpath = None
if confs:
compile_classpath = ClasspathProducts(self.get_options().pants_workdir)
self.resolve(executor=executor,
targets=targets,
classpath_products=compile_classpath,
confs=confs,
extra_args=())
return compile_classpath
def runner(self, jvm_options=None, args=None, executor=None):
"""Creates an ivy commandline client runner for the given args."""
args = args or []
if self._ivy_settings and '-settings' not in args:
args = ['-settings', self._ivy_settings] + args
options = list(jvm_options) if jvm_options else []
if self._ivy_resolution_cache_dir and '-cache' not in args:
# TODO(John Sirois): Currently this is a magic property to support hand-crafted <caches/> in
# ivysettings.xml. Ideally we'd support either simple -caches or these hand-crafted cases
# instead of just hand-crafted. Clean this up by taking over ivysettings.xml and generating
# it from BUILD constructs.
options += [
'-Divy.cache.dir={}'.format(self._ivy_resolution_cache_dir)
]
options += self._extra_jvm_options
executor = executor or SubprocessExecutor(DistributionLocator.cached())
if not isinstance(executor, Executor):
raise ValueError(
'The executor argument must be an Executor instance, given {} of type {}'
.format(executor, type(executor)))
return executor.runner(classpath=self._classpath,
main='org.apache.ivy.Main',
jvm_options=options,
args=args)
def resolve_jars(self, targets):
# TODO: Why is this computed directly here instead of taking from the actual product
# computed by the {Ivy,Coursier}Resolve task?
executor = SubprocessExecutor(DistributionLocator.cached())
confs = []
if self.get_options().libraries:
confs.append("default")
if self.get_options().libraries_sources:
confs.append("sources")
if self.get_options().libraries_javadocs:
confs.append("javadoc")
compile_classpath = None
if confs:
compile_classpath = ClasspathProducts(
self.get_options().pants_workdir)
CoursierMixin.resolve(
self,
targets,
compile_classpath,
sources=self.get_options().libraries_sources,
javadoc=self.get_options().libraries_javadocs,
executor=executor,
)
return compile_classpath
def execute(self):
targets = self.context.targets(predicate=self._is_benchmark)
if not targets:
raise TaskError('No jvm targets specified for benchmarking.')
# For rewriting JDK classes to work, the JAR file has to be listed specifically in
# the JAR manifest as something that goes in the bootclasspath.
# The MANIFEST list a jar 'allocation.jar' this is why we have to rename it
agent_tools_classpath = self.tool_classpath('benchmark-agent')
agent_jar = agent_tools_classpath[0]
allocation_jar = os.path.join(os.path.dirname(agent_jar),
"allocation.jar")
# TODO(Steve Gury): Find a solution to avoid copying the jar every run and being resilient
# to version upgrade
shutil.copyfile(agent_jar, allocation_jar)
os.environ['ALLOCATION_JAR'] = str(allocation_jar)
benchmark_tools_classpath = self.tool_classpath('benchmark-tool')
# Collect a transitive classpath for the benchmark targets.
classpath = self.classpath(targets, benchmark_tools_classpath)
java_executor = SubprocessExecutor(DistributionLocator.cached())
exit_code = execute_java(classpath=classpath,
main=self._CALIPER_MAIN,
jvm_options=self.jvm_options,
args=self.args,
workunit_factory=self.context.new_workunit,
workunit_name='caliper',
workunit_labels=[WorkUnitLabel.RUN],
executor=java_executor)
if exit_code != 0:
raise TaskError('java {} ... exited non-zero ({})'.format(
self._CALIPER_MAIN, exit_code))
def __init__(self,
classpath,
java_executor=None,
ivy_settings=None,
ivy_cache_dir=None):
"""Configures an ivy wrapper for the ivy distribution at the given classpath."""
self._classpath = maybe_list(classpath)
self._java = java_executor or SubprocessExecutor()
if not isinstance(self._java, Executor):
raise ValueError(
'java_executor must be an Executor instance, given %s of type %s'
% (self._java, type(self._java)))
self._ivy_settings = ivy_settings
if self._ivy_settings and not isinstance(self._ivy_settings,
Compatibility.string):
raise ValueError(
'ivy_settings must be a string, given %s of type %s' %
(self._ivy_settings, type(self._ivy_settings)))
self._ivy_cache_dir = ivy_cache_dir
if self._ivy_cache_dir and not isinstance(self._ivy_cache_dir,
Compatibility.string):
raise ValueError(
'ivy_cache_dir must be a string, given %s of type %s' %
(self._ivy_cache_dir, type(self._ivy_cache_dir)))
def execute_junit_runner(self, content):
# Create the temporary base test directory
test_rel_path = 'tests/java/org/pantsbuild/foo'
test_abs_path = os.path.join(self.build_root, test_rel_path)
self.create_dir(test_rel_path)
# Generate the temporary java test source code.
test_java_file_rel_path = os.path.join(test_rel_path, 'FooTest.java')
test_java_file_abs_path = os.path.join(self.build_root, test_java_file_rel_path)
self.create_file(test_java_file_rel_path, content)
# Invoke ivy to resolve classpath for junit.
distribution = Distribution.cached(jdk=True)
executor = SubprocessExecutor(distribution=distribution)
classpath_file_abs_path = os.path.join(test_abs_path, 'junit.classpath')
with subsystem_instance(IvySubsystem) as ivy_subsystem:
ivy = Bootstrapper(ivy_subsystem=ivy_subsystem).ivy()
ivy.execute(args=['-cachepath', classpath_file_abs_path,
'-dependency', 'junit', 'junit-dep', '4.10'], executor=executor)
with open(classpath_file_abs_path) as fp:
classpath = fp.read()
# Now directly invoking javac to compile the test java code into java class
# so later we can inject the class into products mapping for JUnitRun to execute
# the test on.
javac = distribution.binary('javac')
subprocess.check_call(
[javac, '-d', test_abs_path, '-cp', classpath, test_java_file_abs_path])
# Create a java_tests target and a synthetic resource target.
java_tests = self.create_library(test_rel_path, 'java_tests', 'foo_test', ['FooTest.java'])
resources = self.make_target('some_resources', Resources)
# Set the context with the two targets, one java_tests target and
# one synthetic resources target.
# The synthetic resources target is to make sure we won't regress
# in the future with bug like https://github.com/pantsbuild/pants/issues/508. Note
# in that bug, the resources target must be the first one in the list.
context = self.context(target_roots=[resources, java_tests])
# Before we run the task, we need to inject the "classes_by_target" with
# the compiled test java classes that JUnitRun will know which test
# classes to execute. In a normal run, this "classes_by_target" will be
# populated by java compiling step.
class_products = context.products.get_data(
'classes_by_target', lambda: defaultdict(MultipleRootedProducts))
java_tests_products = MultipleRootedProducts()
java_tests_products.add_rel_paths(test_abs_path, ['FooTest.class'])
class_products[java_tests] = java_tests_products
# Also we need to add the FooTest.class's classpath to the compile_classpath
# products data mapping so JUnitRun will be able to add that into the final
# classpath under which the junit will be executed.
self.populate_compile_classpath(context=context, classpath=[test_abs_path])
# Finally execute the task.
self.execute(context)
def setUp(self):
self.jarjar = '/not/really/jarjar.jar'
init_subsystem(DistributionLocator)
executor = SubprocessExecutor(DistributionLocator.cached())
self.shader = Shader(jarjar_classpath=[self.jarjar],
executor=executor,
binary_package_excludes=['javax'])
self.output_jar = '/not/really/shaded.jar'
def execute_java_for_targets(self, targets, *args, **kwargs):
"""Execute java for targets using the test mixin spawn and wait.
Activates timeouts and other common functionality shared among tests.
"""
distribution = self.preferred_jvm_distribution_for_targets(targets)
actual_executor = kwargs.get('executor') or SubprocessExecutor(distribution)
return self._spawn_and_wait(*args, executor=actual_executor, distribution=distribution, **kwargs)
def execute_tool(self, classpath, main, args=None):
init_subsystem(DistributionLocator)
executor = SubprocessExecutor(DistributionLocator.cached())
process = executor.spawn(classpath, main, args=args,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = process.communicate()
self.assertEqual(0, process.returncode)
self.assertEqual('', err.strip().decode())
yield out.decode()
def execute_java_for_coverage(self, targets, *args, **kwargs):
"""Execute java for targets directly and don't use the test mixin.
This execution won't be wrapped with timeouts and other test mixin code common
across test targets. Used for coverage instrumentation.
"""
distribution = self.preferred_jvm_distribution_for_targets(targets)
actual_executor = SubprocessExecutor(distribution)
return distribution.execute_java(*args, executor=actual_executor, **kwargs)
def create(cls, context, executor=None):
"""Creates and returns a new Shader.
:param Executor executor: Optional java executor to run jarjar with.
"""
if executor is None:
executor = SubprocessExecutor(DistributionLocator.cached())
classpath = cls.global_instance().tool_classpath_from_products(context.products, 'jarjar',
cls.options_scope)
return Shader(classpath, executor)
def __init__(self, jarjar, executor=None):
"""Creates a `Shader` the will use the given `jarjar` jar to create shaded jars.
:param unicode jarjar: The path to the jarjar jar.
:param executor: An optional java `Executor` to use to create shaded jar files. Defaults to a
`SubprocessExecutor` that uses the default java distribution.
"""
self._jarjar = jarjar
self._executor = executor or SubprocessExecutor()
self._system_packages = None
def execute_java_for_targets(self,
targets,
executor=None,
*args,
**kwargs):
distribution = self.preferred_jvm_distribution_for_targets(targets)
self._executor = executor or SubprocessExecutor(distribution)
return distribution.execute_java(*args,
executor=self._executor,
**kwargs)
def _spawn(self, distribution, executor=None, *args, **kwargs):
"""Returns a processhandler to a process executing java.
:param Executor executor: the java subprocess executor to use. If not specified, construct
using the distribution.
:param Distribution distribution: The JDK or JRE installed.
:rtype: ProcessHandler
"""
actual_executor = executor or SubprocessExecutor(distribution)
return distribution.execute_java_async(*args, executor=actual_executor, **kwargs)
def do_test_jre_env_var(self, env_var, env_value, scrubbed=True):
with self.jre(env_var=env_var) as jre:
executor = SubprocessExecutor(Distribution(bin_path=jre))
with environment_as(**{env_var: env_value}):
self.assertEqual(env_value, os.getenv(env_var))
process = executor.spawn(classpath=['dummy/classpath'],
main='dummy.main',
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
_, stderr = process.communicate()
self.assertEqual(0, process.returncode)
self.assertEqual('' if scrubbed else env_value, stderr.decode().strip())
def create_javadoc_command(self, classpath, gendir, *targets):
sources = []
for target in targets:
sources.extend(target.sources_relative_to_buildroot())
if not sources:
return None
# Without a JDK/tools.jar we have no javadoc tool and cannot proceed, so check/acquire early.
jdk = DistributionLocator.cached(jdk=True)
tool_classpath = jdk.find_libs(['tools.jar'])
args = ['-quiet',
'-encoding', 'UTF-8',
'-notimestamp',
'-use',
'-Xmaxerrs', '10000', # the default is 100
'-Xmaxwarns', '10000', # the default is 100
'-d', gendir]
# Always provide external linking for java API
offlinelinks = {'http://download.oracle.com/javase/8/docs/api/'}
def link(target):
for jar in target.jar_dependencies:
if jar.apidocs:
offlinelinks.add(jar.apidocs)
for target in targets:
target.walk(link, lambda t: isinstance(t, (JvmTarget, JarLibrary)))
for link in offlinelinks:
args.extend(['-linkoffline', link, link])
args.extend(self.args)
javadoc_classpath_file = os.path.join(self.workdir, f'{os.path.basename(gendir)}.classpath')
with open(javadoc_classpath_file, 'w') as f:
f.write('-classpath ')
f.write(':'.join(classpath))
args.extend([f'@{javadoc_classpath_file}'])
javadoc_sources_file = os.path.join(self.workdir, f'{os.path.basename(gendir)}.source.files')
with open(javadoc_sources_file, 'w') as f:
f.write('\n'.join(sources))
args.extend([f'@{javadoc_sources_file}'])
java_executor = SubprocessExecutor(jdk)
runner = java_executor.runner(jvm_options=self.jvm_options,
classpath=tool_classpath,
main='com.sun.tools.javadoc.Main',
args=args)
return runner.command
def _get_runner(classpath, main, jvm_options, args, executor,
cwd, distribution,
create_synthetic_jar, synthetic_jar_dir):
"""Gets the java runner for execute_java and execute_java_async."""
executor = executor or SubprocessExecutor(distribution)
safe_cp = classpath
if create_synthetic_jar:
safe_cp = safe_classpath(classpath, synthetic_jar_dir)
logger.debug('Bundling classpath {} into {}'.format(':'.join(classpath), safe_cp))
return executor.runner(safe_cp, main, args=args, jvm_options=jvm_options, cwd=cwd)
def create_java_executor(self):
"""Create java executor that uses this task's ng daemon, if allowed.
Call only in execute() or later. TODO: Enforce this.
"""
if self.nailgun_is_enabled and self.get_options().ng_daemons:
classpath = os.pathsep.join(self.tool_classpath('nailgun-server'))
client = NailgunExecutor(self._executor_workdir,
classpath,
distribution=self._dist)
else:
client = SubprocessExecutor(self._dist)
return client
def execute(self):
if self.goal not in JvmPrepCommand.goals():
raise AssertionError(
'Got goal "{}". Expected goal to be one of {}'.format(
self.goal, JvmPrepCommand.goals()))
targets = self.context.targets(postorder=True,
predicate=self.runnable_prep_cmd)
compile_classpath = self.context.products.get_data('compile_classpath')
classpath_products = self.context.products.get_data(
'runtime_classpath', compile_classpath.copy)
with self.context.new_workunit(name='jvm_prep_command',
labels=[WorkUnitLabel.PREP
]) as workunit:
for target in targets:
distribution = JvmPlatform.preferred_jvm_distribution(
[target.platform])
executor = SubprocessExecutor(distribution)
mainclass = target.payload.get_field_value('mainclass')
args = target.payload.get_field_value('args', [])
target_jvm_options = target.payload.get_field_value(
'jvm_options', [])
cp = list(
ClasspathUtil.classpath(target.closure(),
classpath_products))
if not cp:
raise TaskError(
'target {} has no classpath. (Add dependencies= parameter?'
.format(target.address.spec))
self.context.log.info('Running prep command for {}'.format(
target.address.spec))
returncode = distribution.execute_java(
executor=executor,
classpath=cp,
main=mainclass,
jvm_options=target_jvm_options,
args=args,
workunit_factory=self.context.new_workunit,
workunit_name='run',
workunit_labels=[WorkUnitLabel.PREP],
)
workunit.set_outcome(
WorkUnit.FAILURE if returncode else WorkUnit.SUCCESS)
if returncode:
raise TaskError(
'RunJvmPrepCommand failed to run {}'.format(mainclass))
def post_fork_child(self, fingerprint, jvm_options, classpath, stdout, stderr):
"""Post-fork() child callback for ProcessManager.daemon_spawn()."""
java = SubprocessExecutor(self._distribution)
subproc = java.spawn(classpath=classpath,
main='com.martiansoftware.nailgun.NGServer',
jvm_options=jvm_options,
args=[':0'],
stdin=safe_open('/dev/null', 'r'),
stdout=safe_open(self._ng_stdout, 'w'),
stderr=safe_open(self._ng_stderr, 'w'),
close_fds=True)
self.write_pid(subproc.pid)
def create_java_executor(self):
"""Create java executor that uses this task's ng daemon, if allowed.
Call only in execute() or later. TODO: Enforce this.
"""
if self.context.options.nailgun_daemon:
classpath = os.pathsep.join(
self.tool_classpath(self._nailgun_bootstrap_key))
client = NailgunExecutor(self._executor_workdir,
classpath,
distribution=self._dist)
else:
client = SubprocessExecutor(self._dist)
return client
def create(cls, context, executor=None):
"""Creates and returns a new Shader.
:param Executor executor: Optional java executor to run jarjar with.
"""
if executor is None:
executor = SubprocessExecutor(DistributionLocator.cached())
classpath = cls.global_instance().tool_classpath_from_products(
context.products, 'jarjar', cls.options_scope)
if len(classpath) != 1:
raise cls.Error(
'JarJar classpath is expected to have exactly one jar!\nclasspath = {}'
.format(':'.join(classpath)))
jarjar = classpath[0]
return Shader(jarjar, executor)
