def execute(
self,
jvm_options=None,
args=None,
executor=None,
workunit_factory=None,
workunit_name=None,
workunit_labels=None,
):
"""Executes the ivy commandline client with the given args.
Raises Ivy.Error if the command fails for any reason.
:param executor: Java executor to run ivy with.
"""
# NB(gmalmquist): It should be OK that we can't declare a subsystem_dependency in this file
# (because it's just a plain old object), because Ivy is only constructed by Bootstrapper, which
# makes an explicit call to IvySubsystem.global_instance() in its constructor, which in turn has
# a declared dependency on DistributionLocator.
executor = executor or SubprocessExecutor(DistributionLocator.cached())
runner = self.runner(jvm_options=jvm_options,
args=args,
executor=executor)
try:
with self.resolution_lock():
result = util.execute_runner(runner, workunit_factory,
workunit_name, workunit_labels)
if result != 0:
raise self.Error(
"Ivy command failed with exit code {}{}".format(
result, ": " + " ".join(args) if args else ""))
except executor.Error as e:
raise self.Error("Problem executing ivy: {}".format(e))
def exec_ivy(cls, ivy, confs, ivyxml, args, jvm_options, executor,
workunit_name, workunit_factory):
ivy = ivy or Bootstrapper.default_ivy()
ivy_args = ['-ivy', ivyxml]
ivy_args.append('-confs')
ivy_args.extend(confs)
ivy_args.extend(args)
ivy_jvm_options = list(jvm_options)
# Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng.
ivy_jvm_options.append('-Dsun.io.useCanonCaches=false')
runner = ivy.runner(jvm_options=ivy_jvm_options,
args=ivy_args,
executor=executor)
try:
result = execute_runner(runner,
workunit_factory=workunit_factory,
workunit_name=workunit_name)
if result != 0:
raise IvyUtils.IvyError(
'Ivy returned {result}. cmd={cmd}'.format(result=result,
cmd=runner.cmd))
except runner.executor.Error as e:
raise IvyUtils.IvyError(e)
def shade_jar(self, shading_rules, jar_path):
"""Shades a jar using the shading rules from the given jvm_binary.
This *overwrites* the existing jar file at ``jar_path``.
:param shading_rules: predefined rules for shading
:param jar_path: The filepath to the jar that should be shaded.
"""
self.context.log.debug('Shading {}.'.format(jar_path))
with temporary_dir() as tempdir:
output_jar = os.path.join(tempdir, os.path.basename(jar_path))
with self.shader.binary_shader_for_rules(
output_jar, jar_path, shading_rules) as shade_runner:
result = execute_runner(
shade_runner,
workunit_factory=self.context.new_workunit,
workunit_name='jarjar')
if result != 0:
raise TaskError(
'Shading tool failed to shade {0} (error code {1})'.
format(jar_path, result))
if not os.path.exists(output_jar):
raise TaskError(
'Shading tool returned success for {0}, but '
'the output jar was not found at {1}'.format(
jar_path, output_jar))
atomic_copy(output_jar, jar_path)
return jar_path
def execute(
self,
jvm_options=None,
args=None,
executor=None,
workunit_factory=None,
workunit_name=None,
workunit_labels=None,
):
"""Executes the ivy commandline client with the given args.
Raises Ivy.Error if the command fails for any reason.
:param executor: Java executor to run ivy with.
"""
# NB(gmalmquist): It should be OK that we can't declare a subsystem_dependency in this file
# (because it's just a plain old object), because Ivy is only constructed by Bootstrapper, which
# makes an explicit call to IvySubsystem.global_instance() in its constructor, which in turn has
# a declared dependency on DistributionLocator.
executor = executor or SubprocessExecutor(DistributionLocator.cached())
runner = self.runner(jvm_options=jvm_options, args=args, executor=executor)
try:
with self.resolution_lock:
result = util.execute_runner(runner, workunit_factory, workunit_name, workunit_labels)
if result != 0:
raise self.Error(
"Ivy command failed with exit code {}{}".format(result, ": " + " ".join(args) if args else "")
)
except executor.Error as e:
raise self.Error("Problem executing ivy: {}".format(e))
def exec_ivy(cls, ivy, confs, ivyxml, args,
jvm_options,
executor,
workunit_name,
workunit_factory):
"""
:API: public
"""
ivy = ivy or Bootstrapper.default_ivy()
ivy_args = ['-ivy', ivyxml]
ivy_args.append('-confs')
ivy_args.extend(confs)
ivy_args.extend(args)
ivy_jvm_options = list(jvm_options)
# Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng.
ivy_jvm_options.append('-Dsun.io.useCanonCaches=false')
runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor)
try:
with ivy.resolution_lock:
result = execute_runner(runner, workunit_factory=workunit_factory,
workunit_name=workunit_name)
if result != 0:
raise IvyUtils.IvyError('Ivy returned {result}. cmd={cmd}'.format(result=result, cmd=runner.cmd))
except runner.executor.Error as e:
raise IvyUtils.IvyError(e)
def exec_ivy(self,
target_workdir,
targets,
args,
confs=None,
ivy=None,
workunit_name='ivy',
workunit_factory=None,
symlink_ivyxml=False,
jars=None):
ivy = ivy or Bootstrapper.default_ivy()
if not isinstance(ivy, Ivy):
raise ValueError('The ivy argument supplied must be an Ivy instance, given %s of type %s'
% (ivy, type(ivy)))
ivyxml = os.path.join(target_workdir, 'ivy.xml')
if not jars:
jars, excludes = self._calculate_classpath(targets)
else:
excludes = set()
ivy_args = ['-ivy', ivyxml]
confs_to_resolve = confs or ['default']
ivy_args.append('-confs')
ivy_args.extend(confs_to_resolve)
ivy_args.extend(args)
if not self._transitive:
ivy_args.append('-notransitive')
ivy_args.extend(self._args)
def safe_link(src, dest):
try:
os.unlink(dest)
except OSError as e:
if e.errno != errno.ENOENT:
raise
os.symlink(src, dest)
with IvyUtils.ivy_lock:
self._generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve)
runner = ivy.runner(jvm_options=self._jvm_options, args=ivy_args)
try:
result = util.execute_runner(runner,
workunit_factory=workunit_factory,
workunit_name=workunit_name)
# Symlink to the current ivy.xml file (useful for IDEs that read it).
if symlink_ivyxml:
ivyxml_symlink = os.path.join(self._workdir, 'ivy.xml')
safe_link(ivyxml, ivyxml_symlink)
if result != 0:
raise TaskError('Ivy returned %d' % result)
except runner.executor.Error as e:
raise TaskError(e)
def _bootstrap_shaded_jvm_tool(self, key, scope, tools, main, custom_rules=None):
shaded_jar = os.path.join(self._tool_cache_path,
'shaded_jars', scope, key, '{}.jar'.format(main))
targets = list(self._resolve_tool_targets(tools, key, scope))
fingerprint_strategy = ShadedToolFingerprintStrategy(key, scope, main,
custom_rules=custom_rules)
with self.invalidated(targets,
# We're the only dependent in reality since we shade.
invalidate_dependents=False,
fingerprint_strategy=fingerprint_strategy) as invalidation_check:
if not invalidation_check.invalid_vts and os.path.exists(shaded_jar):
return [shaded_jar]
# Ensure we have a single binary jar we can shade.
binary_jar = os.path.join(self._tool_cache_path,
'binary_jars', scope, key, '{}.jar'.format(main))
safe_mkdir_for(binary_jar)
classpath = self._bootstrap_classpath(key, targets)
if len(classpath) == 1:
shutil.copy(classpath[0], binary_jar)
else:
with self.open_jar(binary_jar) as jar:
for classpath_jar in classpath:
jar.writejar(classpath_jar)
jar.main(main)
# Now shade the binary jar and return that single jar as the safe tool classpath.
safe_mkdir_for(shaded_jar)
with self.shader.binary_shader(shaded_jar,
main,
binary_jar,
custom_rules=custom_rules,
jvm_options=self.get_options().jvm_options) as shader:
try:
result = util.execute_runner(shader,
workunit_factory=self.context.new_workunit,
workunit_name='shade-{}'.format(key))
if result != 0:
raise TaskError("Shading of tool '{key}' with main class {main} for {scope} failed "
"with exit code {result}, command run was:\n\t{cmd}"
.format(key=key, main=main, scope=scope, result=result, cmd=shader.cmd))
except Executor.Error as e:
raise TaskError("Shading of tool '{key}' with main class {main} for {scope} failed "
"with: {exception}".format(key=key, main=main, scope=scope, exception=e))
if self.artifact_cache_writes_enabled():
tool_vts = self.tool_vts(invalidation_check)
self.update_artifact_cache([(tool_vts, [shaded_jar])])
return [shaded_jar]
def execute(self, jvm_options=None, args=None, executor=None,
workunit_factory=None, workunit_name=None, workunit_labels=None):
"""Executes the ivy commandline client with the given args.
Raises Ivy.Error if the command fails for any reason.
"""
runner = self.runner(jvm_options=jvm_options, args=args, executor=executor)
try:
result = util.execute_runner(runner, workunit_factory, workunit_name, workunit_labels)
if result != 0:
raise self.Error('Ivy command failed with exit code %d%s'
% (result, ': ' + ' '.join(args) if args else ''))
except self._java.Error as e:
raise self.Error('Problem executing ivy: %s' % e)
def execute(self, jvm_options=None, args=None, executor=None,
workunit_factory=None, workunit_name=None, workunit_labels=None):
"""Executes the ivy commandline client with the given args.
Raises Ivy.Error if the command fails for any reason.
"""
runner = self.runner(jvm_options=jvm_options, args=args, executor=executor)
try:
result = util.execute_runner(runner, workunit_factory, workunit_name, workunit_labels)
if result != 0:
raise self.Error('Ivy command failed with exit code %d%s'
% (result, ': ' + ' '.join(args) if args else ''))
except self._java.Error as e:
raise self.Error('Problem executing ivy: %s' % e)
def _exec_ivy(self,
target_workdir,
targets,
args,
executor=None,
confs=None,
ivy=None,
workunit_name='ivy',
use_soft_excludes=False,
resolve_hash_name=None):
ivy_jvm_options = self.get_options().jvm_options[:]
# Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng.
ivy_jvm_options.append('-Dsun.io.useCanonCaches=false')
ivy = ivy or Bootstrapper.default_ivy()
ivyxml = os.path.join(target_workdir, 'ivy.xml')
ivy_args = ['-ivy', ivyxml]
confs_to_resolve = confs or ('default', )
ivy_args.append('-confs')
ivy_args.extend(confs_to_resolve)
ivy_args.extend(args)
# TODO(John Sirois): merge the code below into IvyUtils or up here; either way, better
# diagnostics can be had in `IvyUtils.generate_ivy` if this is done.
# See: https://github.com/pantsbuild/pants/issues/2239
try:
jars, excludes = IvyUtils.calculate_classpath(
targets, gather_excludes=not use_soft_excludes)
with IvyUtils.ivy_lock:
IvyUtils.generate_ivy(targets, jars, excludes, ivyxml,
confs_to_resolve, resolve_hash_name)
runner = ivy.runner(jvm_options=ivy_jvm_options,
args=ivy_args,
executor=executor)
try:
result = execute_runner(
runner,
workunit_factory=self.context.new_workunit,
workunit_name=workunit_name)
if result != 0:
raise self.Error(
'Ivy returned {result}. cmd={cmd}'.format(
result=result, cmd=runner.cmd))
except runner.executor.Error as e:
raise self.Error(e)
except IvyUtils.IvyError as e:
raise self.Error('Failed to prepare ivy resolve: {}'.format(e))
def _call_coursier(self, cmd_args, coursier_jar, output_fn, executor):
runner = executor.runner(
classpath=[coursier_jar],
main='coursier.cli.Coursier',
jvm_options=self.get_options().jvm_options,
args=cmd_args)
labels = [WorkUnitLabel.COMPILER] if self.get_options().report else [WorkUnitLabel.TOOL]
return_code = util.execute_runner(runner, self.context.new_workunit, 'coursier', labels)
if return_code:
raise TaskError('The coursier process exited non-zero: {0}'.format(return_code))
with open(output_fn, 'r') as f:
return json.loads(f.read())
def _call_coursier(self, cmd_args, coursier_jar, output_fn, executor):
runner = executor.runner(
classpath=[coursier_jar],
main='coursier.cli.Coursier',
jvm_options=self.get_options().jvm_options,
args=cmd_args)
labels = [WorkUnitLabel.COMPILER] if self.get_options().report else [WorkUnitLabel.TOOL]
return_code = util.execute_runner(runner, self.context.new_workunit, 'coursier', labels)
if return_code:
raise TaskError('The coursier process exited non-zero: {0}'.format(return_code))
with open(output_fn, 'r') as f:
return json.loads(f.read())
def exec_ivy(self,
target_workdir,
targets,
args,
executor=None,
confs=None,
ivy=None,
workunit_name='ivy',
jars=None,
use_soft_excludes=False,
resolve_hash_name=None):
ivy_jvm_options = copy.copy(self.get_options().jvm_options)
# Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng.
ivy_jvm_options.append('-Dsun.io.useCanonCaches=false')
ivy = ivy or Bootstrapper.default_ivy()
ivyxml = os.path.join(target_workdir, 'ivy.xml')
if not jars:
jars, excludes = IvyUtils.calculate_classpath(
targets, gather_excludes=not use_soft_excludes)
else:
excludes = set()
ivy_args = ['-ivy', ivyxml]
confs_to_resolve = confs or ['default']
ivy_args.append('-confs')
ivy_args.extend(confs_to_resolve)
ivy_args.extend(args)
with IvyUtils.ivy_lock:
IvyUtils.generate_ivy(targets, jars, excludes, ivyxml,
confs_to_resolve, resolve_hash_name)
runner = ivy.runner(jvm_options=ivy_jvm_options,
args=ivy_args,
executor=executor)
try:
result = execute_runner(
runner,
workunit_factory=self.context.new_workunit,
workunit_name=workunit_name)
if result != 0:
raise TaskError('Ivy returned {result}. cmd={cmd}'.format(
result=result, cmd=runner.cmd))
except runner.executor.Error as e:
raise TaskError(e)
def _exec_ivy(
self,
target_workdir,
targets,
args,
executor=None,
confs=None,
ivy=None,
workunit_name="ivy",
use_soft_excludes=False,
resolve_hash_name=None,
):
ivy_jvm_options = self.get_options().jvm_options[:]
# Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng.
ivy_jvm_options.append("-Dsun.io.useCanonCaches=false")
ivy = ivy or Bootstrapper.default_ivy()
ivyxml = os.path.join(target_workdir, "ivy.xml")
ivy_args = ["-ivy", ivyxml]
confs_to_resolve = confs or ("default",)
ivy_args.append("-confs")
ivy_args.extend(confs_to_resolve)
ivy_args.extend(args)
# TODO(John Sirois): merge the code below into IvyUtils or up here; either way, better
# diagnostics can be had in `IvyUtils.generate_ivy` if this is done.
# See: https://github.com/pantsbuild/pants/issues/2239
try:
jars, excludes = IvyUtils.calculate_classpath(targets, gather_excludes=not use_soft_excludes)
with IvyUtils.ivy_lock:
IvyUtils.generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve, resolve_hash_name)
runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor)
try:
result = execute_runner(
runner, workunit_factory=self.context.new_workunit, workunit_name=workunit_name
)
if result != 0:
raise self.Error("Ivy returned {result}. cmd={cmd}".format(result=result, cmd=runner.cmd))
except runner.executor.Error as e:
raise self.Error(e)
except IvyUtils.IvyError as e:
raise self.Error("Failed to prepare ivy resolve: {}".format(e))
def exec_ivy(self,
target_workdir,
targets,
args,
executor=None,
confs=None,
ivy=None,
workunit_name='ivy',
jars=None,
use_soft_excludes=False,
resolve_hash_name=None):
ivy_jvm_options = copy.copy(self.get_options().jvm_options)
# Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng.
ivy_jvm_options.append('-Dsun.io.useCanonCaches=false')
ivy = ivy or Bootstrapper.default_ivy()
ivyxml = os.path.join(target_workdir, 'ivy.xml')
if not jars:
automatic_excludes = self.get_options().automatic_excludes
jars, excludes = IvyUtils.calculate_classpath(targets,
gather_excludes=not use_soft_excludes,
automatic_excludes=automatic_excludes)
else:
excludes = set()
ivy_args = ['-ivy', ivyxml]
confs_to_resolve = confs or ['default']
ivy_args.append('-confs')
ivy_args.extend(confs_to_resolve)
ivy_args.extend(args)
with IvyUtils.ivy_lock:
IvyUtils.generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve, resolve_hash_name)
runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor)
try:
result = execute_runner(runner, workunit_factory=self.context.new_workunit,
workunit_name=workunit_name)
if result != 0:
raise TaskError('Ivy returned {result}. cmd={cmd}'.format(result=result, cmd=runner.cmd))
except runner.executor.Error as e:
raise TaskError(e)
def shade_jar(self, shading_rules, jar_path):
"""Shades a jar using the shading rules from the given jvm_binary.
This *overwrites* the existing jar file at ``jar_path``.
:param shading_rules: predefined rules for shading
:param jar_path: The filepath to the jar that should be shaded.
"""
self.context.log.debug('Shading {}.'.format(jar_path))
with temporary_dir() as tempdir:
output_jar = os.path.join(tempdir, os.path.basename(jar_path))
with self.shader.binary_shader_for_rules(output_jar, jar_path, shading_rules) as shade_runner:
result = execute_runner(shade_runner, workunit_factory=self.context.new_workunit,
workunit_name='jarjar')
if result != 0:
raise TaskError('Shading tool failed to shade {0} (error code {1})'.format(jar_path,
result))
if not os.path.exists(output_jar):
raise TaskError('Shading tool returned success for {0}, but '
'the output jar was not found at {1}'.format(jar_path, output_jar))
atomic_copy(output_jar, jar_path)
return jar_path
def _bootstrap_shaded_jvm_tool(self, jvm_tool, targets):
fingerprint_strategy = ShadedToolFingerprintStrategy(
jvm_tool.main, custom_rules=jvm_tool.custom_rules)
with self.invalidated(
targets,
# We're the only dependent in reality since we shade.
invalidate_dependents=False,
fingerprint_strategy=fingerprint_strategy
) as invalidation_check:
# If there are no vts, then there are no resolvable targets, so we exit early with an empty
# classpath. This supports the optional tool classpath case.
if not invalidation_check.all_vts:
return []
tool_vts = self.tool_vts(invalidation_check)
jar_name = '{main}-{hash}.jar'.format(main=jvm_tool.main,
hash=tool_vts.cache_key.hash)
shaded_jar = os.path.join(self._tool_cache_path, 'shaded_jars',
jar_name)
if not invalidation_check.invalid_vts and os.path.exists(
shaded_jar):
return [shaded_jar]
# Ensure we have a single binary jar we can shade.
binary_jar = os.path.join(self._tool_cache_path, 'binary_jars',
jar_name)
safe_mkdir_for(binary_jar)
classpath = self._bootstrap_classpath(jvm_tool, targets)
if len(classpath) == 1:
shutil.copy(classpath[0], binary_jar)
else:
with self.open_jar(binary_jar) as jar:
for classpath_jar in classpath:
jar.writejar(classpath_jar)
jar.main(jvm_tool.main)
# Now shade the binary jar and return that single jar as the safe tool classpath.
safe_mkdir_for(shaded_jar)
with self.shader.binary_shader(
shaded_jar,
jvm_tool.main,
binary_jar,
custom_rules=jvm_tool.custom_rules,
jvm_options=self.get_options().jvm_options) as shader:
try:
result = util.execute_runner(
shader,
workunit_factory=self.context.new_workunit,
workunit_name='shade-{}'.format(jvm_tool.key))
if result != 0:
raise TaskError(
"Shading of tool '{key}' with main class {main} for {scope} failed "
"with exit code {result}, command run was:\n\t{cmd}"
.format(key=jvm_tool.key,
main=jvm_tool.main,
scope=jvm_tool.scope,
result=result,
cmd=shader.cmd))
except Executor.Error as e:
raise TaskError(
"Shading of tool '{key}' with main class {main} for {scope} failed "
"with: {exception}".format(key=jvm_tool.key,
main=jvm_tool.main,
scope=jvm_tool.scope,
exception=e))
if self.artifact_cache_writes_enabled():
self.update_artifact_cache([(tool_vts, [shaded_jar])])
return [shaded_jar]
def _bootstrap_shaded_jvm_tool(self, jvm_tool, targets):
fingerprint_strategy = ShadedToolFingerprintStrategy(jvm_tool.main,
custom_rules=jvm_tool.custom_rules)
with self.invalidated(targets,
# We're the only dependent in reality since we shade.
invalidate_dependents=False,
fingerprint_strategy=fingerprint_strategy) as invalidation_check:
# If there are no vts, then there are no resolvable targets, so we exit early with an empty
# classpath. This supports the optional tool classpath case.
if not invalidation_check.all_vts:
return []
tool_vts = self.tool_vts(invalidation_check)
jar_name = '{main}-{hash}.jar'.format(main=jvm_tool.main, hash=tool_vts.cache_key.hash)
shaded_jar = os.path.join(self._tool_cache_path, 'shaded_jars', jar_name)
if not invalidation_check.invalid_vts and os.path.exists(shaded_jar):
return [shaded_jar]
# Ensure we have a single binary jar we can shade.
binary_jar = os.path.join(self._tool_cache_path, 'binary_jars', jar_name)
safe_mkdir_for(binary_jar)
classpath = self._bootstrap_classpath(jvm_tool, targets)
if len(classpath) == 1:
shutil.copy(classpath[0], binary_jar)
else:
with self.open_jar(binary_jar) as jar:
for classpath_jar in classpath:
jar.writejar(classpath_jar)
jar.main(jvm_tool.main)
# Now shade the binary jar and return that single jar as the safe tool classpath.
safe_mkdir_for(shaded_jar)
with self.shader.binary_shader(shaded_jar,
jvm_tool.main,
binary_jar,
custom_rules=jvm_tool.custom_rules,
jvm_options=self.get_options().jvm_options) as shader:
try:
result = util.execute_runner(shader,
workunit_factory=self.context.new_workunit,
workunit_name='shade-{}'.format(jvm_tool.key))
if result != 0:
raise TaskError("Shading of tool '{key}' with main class {main} for {scope} failed "
"with exit code {result}, command run was:\n\t{cmd}"
.format(key=jvm_tool.key,
main=jvm_tool.main,
scope=jvm_tool.scope,
result=result,
cmd=shader.cmd))
except Executor.Error as e:
raise TaskError("Shading of tool '{key}' with main class {main} for {scope} failed "
"with: {exception}".format(key=jvm_tool.key,
main=jvm_tool.main,
scope=jvm_tool.scope,
exception=e))
if self.artifact_cache_writes_enabled():
self.update_artifact_cache([(tool_vts, [shaded_jar])])
return [shaded_jar]