def check_artifact_cache_for(self, invalidation_check): # Ivy resolution is an output dependent on the entire target set, and is not divisible # by target. So we can only cache it keyed by the entire target set. global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts) return [global_vts]
def ivy_resolve(self,
targets,
executor=None,
silent=False,
workunit_name=None,
confs=None,
custom_args=None):
"""Executes an ivy resolve for the relevant subset of the given targets.
Returns the resulting classpath, and the set of relevant targets. Also populates
the 'ivy_resolve_symlink_map' product for jars resulting from the resolve."""
if not targets:
return ([], set())
# NOTE: Always pass all the targets to exec_ivy, as they're used to calculate the name of
# the generated module, which in turn determines the location of the XML report file
# ivy generates. We recompute this name from targets later in order to find that file.
# TODO: This is fragile. Refactor so that we're not computing the name twice.
ivy = Bootstrapper.default_ivy(bootstrap_workunit_factory=self.context.new_workunit)
ivy_workdir = os.path.join(self.context.options.for_global_scope().pants_workdir, 'ivy')
fingerprint_strategy = IvyResolveFingerprintStrategy(confs)
with self.invalidated(targets,
invalidate_dependents=False,
silent=silent,
fingerprint_strategy=fingerprint_strategy) as invalidation_check:
if not invalidation_check.all_vts:
return ([], set())
global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)
# If a report file is not present, we need to exec ivy, even if all the individual
# targets up to date... See https://rbcommons.com/s/twitter/r/2015
report_missing = False
report_confs = confs or ['default']
report_paths = []
for conf in report_confs:
report_path = IvyUtils.xml_report_path(global_vts.targets, conf)
if not os.path.exists(report_path):
report_missing = True
break
else:
report_paths.append(report_path)
target_workdir = os.path.join(ivy_workdir, global_vts.cache_key.hash)
target_classpath_file = os.path.join(target_workdir, 'classpath')
raw_target_classpath_file = target_classpath_file + '.raw'
raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp'
# A common dir for symlinks into the ivy2 cache. This ensures that paths to jars
# in artifact-cached analysis files are consistent across systems.
# Note that we have one global, well-known symlink dir, again so that paths are
# consistent across builds.
symlink_dir = os.path.join(ivy_workdir, 'jars')
# Note that it's possible for all targets to be valid but for no classpath file to exist at
# target_classpath_file, e.g., if we previously built a superset of targets.
if report_missing or invalidation_check.invalid_vts or not os.path.exists(raw_target_classpath_file):
args = ['-cachepath', raw_target_classpath_file_tmp] + (custom_args if custom_args else [])
self.exec_ivy(
target_workdir=target_workdir,
targets=global_vts.targets,
args=args,
executor=executor,
ivy=ivy,
workunit_name=workunit_name,
confs=confs,
use_soft_excludes=self.get_options().soft_excludes)
if not os.path.exists(raw_target_classpath_file_tmp):
raise TaskError('Ivy failed to create classpath file at {}'
.format(raw_target_classpath_file_tmp))
shutil.move(raw_target_classpath_file_tmp, raw_target_classpath_file)
logger.debug('Moved ivy classfile file to {dest}'.format(dest=raw_target_classpath_file))
if self.artifact_cache_writes_enabled():
self.update_artifact_cache([(global_vts, [raw_target_classpath_file])])
else:
logger.debug("Using previously resolved reports: {}".format(report_paths))
# Make our actual classpath be symlinks, so that the paths are uniform across systems.
# Note that we must do this even if we read the raw_target_classpath_file from the artifact
# cache. If we cache the target_classpath_file we won't know how to create the symlinks.
with IvyTaskMixin.symlink_map_lock:
products = self.context.products
existing_symlinks_map = products.get_data('ivy_resolve_symlink_map', lambda: dict())
symlink_map = IvyUtils.symlink_cachepath(
IvySubsystem.global_instance().get_options().cache_dir,
raw_target_classpath_file,
symlink_dir,
target_classpath_file,
existing_symlinks_map)
existing_symlinks_map.update(symlink_map)
with IvyUtils.cachepath(target_classpath_file) as classpath:
stripped_classpath = [path.strip() for path in classpath]
return (stripped_classpath, global_vts.targets)
def tool_vts(self, invalidation_check): # The monolithic shaded tool jar is a single output dependent on the entire target set, and is # not divisible by target. So we can only cache it keyed by the entire target set. return VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)
def ivy_resolve(self,
targets,
executor=None,
symlink_ivyxml=False,
silent=False,
workunit_name=None,
workunit_labels=None):
# NOTE: Always pass all the targets to exec_ivy, as they're used to calculate the name of
# the generated module, which in turn determines the location of the XML report file
# ivy generates. We recompute this name from targets later in order to find that file.
# TODO: This is fragile. Refactor so that we're not computing the name twice.
if executor and not isinstance(executor, Executor):
raise ValueError('The executor must be an Executor instance, given %s of type %s'
% (executor, type(executor)))
ivy = Bootstrapper.default_ivy(java_executor=executor,
bootstrap_workunit_factory=self.context.new_workunit)
if not targets:
return []
ivy_workdir = os.path.join(self.context.config.getdefault('pants_workdir'), 'ivy')
ivy_utils = IvyUtils(config=self.context.config,
options=self.context.options,
log=self.context.log)
fingerprint_strategy = IvyResolveFingerprintStrategy()
with self.invalidated(targets,
invalidate_dependents=True,
silent=silent,
fingerprint_strategy=fingerprint_strategy) as invalidation_check:
global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)
target_workdir = os.path.join(ivy_workdir, global_vts.cache_key.hash)
target_classpath_file = os.path.join(target_workdir, 'classpath')
raw_target_classpath_file = target_classpath_file + '.raw'
raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp'
# A common dir for symlinks into the ivy2 cache. This ensures that paths to jars
# in artifact-cached analysis files are consistent across systems.
# Note that we have one global, well-known symlink dir, again so that paths are
# consistent across builds.
symlink_dir = os.path.join(ivy_workdir, 'jars')
# Note that it's possible for all targets to be valid but for no classpath file to exist at
# target_classpath_file, e.g., if we previously built a superset of targets.
if invalidation_check.invalid_vts or not os.path.exists(raw_target_classpath_file):
args = ['-cachepath', raw_target_classpath_file_tmp]
def exec_ivy():
ivy_utils.exec_ivy(
target_workdir=target_workdir,
targets=targets,
args=args,
ivy=ivy,
workunit_name='ivy',
workunit_factory=self.context.new_workunit,
symlink_ivyxml=symlink_ivyxml)
if workunit_name:
with self.context.new_workunit(name=workunit_name, labels=workunit_labels or []):
exec_ivy()
else:
exec_ivy()
if not os.path.exists(raw_target_classpath_file_tmp):
raise TaskError('Ivy failed to create classpath file at %s'
% raw_target_classpath_file_tmp)
shutil.move(raw_target_classpath_file_tmp, raw_target_classpath_file)
if self.artifact_cache_writes_enabled():
self.update_artifact_cache([(global_vts, [raw_target_classpath_file])])
# Make our actual classpath be symlinks, so that the paths are uniform across systems.
# Note that we must do this even if we read the raw_target_classpath_file from the artifact
# cache. If we cache the target_classpath_file we won't know how to create the symlinks.
symlink_map = IvyUtils.symlink_cachepath(self.context.ivy_home, raw_target_classpath_file,
symlink_dir, target_classpath_file)
with IvyTaskMixin.symlink_map_lock:
all_symlinks_map = self.context.products.get_data('symlink_map') or defaultdict(list)
for path, symlink in symlink_map.items():
all_symlinks_map[os.path.realpath(path)].append(symlink)
self.context.products.safe_create_data('symlink_map', lambda: all_symlinks_map)
with IvyUtils.cachepath(target_classpath_file) as classpath:
stripped_classpath = [path.strip() for path in classpath]
return [path for path in stripped_classpath if ivy_utils.is_classpath_artifact(path)]
def tool_vts(self, invalidation_check): # The monolithic shaded tool jar is a single output dependent on the entire target set, and is # not divisible by target. So we can only cache it keyed by the entire target set. return VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)
def check_artifact_cache_for(self, invalidation_check):
# Ivy resolution is an output dependent on the entire target set, and is not divisible
# by target. So we can only cache it keyed by the entire target set.
global_vts = VersionedTargetSet.from_versioned_targets(
invalidation_check.all_vts)
return [global_vts]
def ivy_resolve(self,
targets,
executor=None,
symlink_ivyxml=False,
silent=False,
workunit_name=None,
workunit_labels=None):
if not targets:
return ([], set())
# NOTE: Always pass all the targets to exec_ivy, as they're used to calculate the name of
# the generated module, which in turn determines the location of the XML report file
# ivy generates. We recompute this name from targets later in order to find that file.
# TODO: This is fragile. Refactor so that we're not computing the name twice.
if executor and not isinstance(executor, Executor):
raise ValueError(
'The executor must be an Executor instance, given %s of type %s'
% (executor, type(executor)))
ivy = Bootstrapper.default_ivy(
java_executor=executor,
bootstrap_workunit_factory=self.context.new_workunit)
ivy_workdir = os.path.join(
self.context.options.for_global_scope().pants_workdir, 'ivy')
ivy_utils = IvyUtils(config=self.context.config, log=self.context.log)
fingerprint_strategy = IvyResolveFingerprintStrategy()
with self.invalidated(targets,
invalidate_dependents=False,
silent=silent,
fingerprint_strategy=fingerprint_strategy
) as invalidation_check:
if not invalidation_check.all_vts:
return ([], set())
global_vts = VersionedTargetSet.from_versioned_targets(
invalidation_check.all_vts)
target_workdir = os.path.join(ivy_workdir,
global_vts.cache_key.hash)
target_classpath_file = os.path.join(target_workdir, 'classpath')
raw_target_classpath_file = target_classpath_file + '.raw'
raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp'
# A common dir for symlinks into the ivy2 cache. This ensures that paths to jars
# in artifact-cached analysis files are consistent across systems.
# Note that we have one global, well-known symlink dir, again so that paths are
# consistent across builds.
symlink_dir = os.path.join(ivy_workdir, 'jars')
# Note that it's possible for all targets to be valid but for no classpath file to exist at
# target_classpath_file, e.g., if we previously built a superset of targets.
if invalidation_check.invalid_vts or not os.path.exists(
raw_target_classpath_file):
args = ['-cachepath', raw_target_classpath_file_tmp]
def exec_ivy():
ivy_utils.exec_ivy(
target_workdir=target_workdir,
targets=global_vts.targets,
args=args,
ivy=ivy,
workunit_name='ivy',
workunit_factory=self.context.new_workunit,
symlink_ivyxml=symlink_ivyxml)
if workunit_name:
with self.context.new_workunit(name=workunit_name,
labels=workunit_labels
or []):
exec_ivy()
else:
exec_ivy()
if not os.path.exists(raw_target_classpath_file_tmp):
raise TaskError(
'Ivy failed to create classpath file at %s' %
raw_target_classpath_file_tmp)
shutil.move(raw_target_classpath_file_tmp,
raw_target_classpath_file)
logger.debug('Copied ivy classfile file to {dest}'.format(
dest=raw_target_classpath_file))
if self.artifact_cache_writes_enabled():
self.update_artifact_cache([(global_vts,
[raw_target_classpath_file])])
# Make our actual classpath be symlinks, so that the paths are uniform across systems.
# Note that we must do this even if we read the raw_target_classpath_file from the artifact
# cache. If we cache the target_classpath_file we won't know how to create the symlinks.
symlink_map = IvyUtils.symlink_cachepath(ivy.ivy_cache_dir,
raw_target_classpath_file,
symlink_dir,
target_classpath_file)
with IvyTaskMixin.symlink_map_lock:
all_symlinks_map = self.context.products.get_data(
'symlink_map') or defaultdict(list)
for path, symlink in symlink_map.items():
all_symlinks_map[os.path.realpath(path)].append(symlink)
self.context.products.safe_create_data('symlink_map',
lambda: all_symlinks_map)
with IvyUtils.cachepath(target_classpath_file) as classpath:
stripped_classpath = [path.strip() for path in classpath]
return ([
path for path in stripped_classpath
if ivy_utils.is_classpath_artifact(path)
], global_vts.targets)
def ivy_resolve(self,
targets,
executor=None,
silent=False,
workunit_name=None,
confs=None,
custom_args=None):
"""Executes an ivy resolve for the relevant subset of the given targets.
:returns: the resulting classpath, and the unique part of the name used for the resolution
report (a hash). Also populates the 'ivy_resolve_symlink_map' product for jars resulting
from the resolve."""
if not targets:
return ([], None)
ivy = Bootstrapper.default_ivy(
bootstrap_workunit_factory=self.context.new_workunit)
ivy_workdir = os.path.join(
self.context.options.for_global_scope().pants_workdir, 'ivy')
fingerprint_strategy = IvyResolveFingerprintStrategy(confs)
with self.invalidated(targets,
invalidate_dependents=False,
silent=silent,
fingerprint_strategy=fingerprint_strategy
) as invalidation_check:
if not invalidation_check.all_vts:
return ([], None)
global_vts = VersionedTargetSet.from_versioned_targets(
invalidation_check.all_vts)
# If a report file is not present, we need to exec ivy, even if all the individual
# targets up to date... See https://rbcommons.com/s/twitter/r/2015
report_missing = False
report_confs = confs or ['default']
report_paths = []
resolve_hash_name = global_vts.cache_key.hash
for conf in report_confs:
report_path = IvyUtils.xml_report_path(resolve_hash_name, conf)
if not os.path.exists(report_path):
report_missing = True
break
else:
report_paths.append(report_path)
target_workdir = os.path.join(ivy_workdir, resolve_hash_name)
target_classpath_file = os.path.join(target_workdir, 'classpath')
raw_target_classpath_file = target_classpath_file + '.raw'
raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp'
# A common dir for symlinks into the ivy2 cache. This ensures that paths to jars
# in artifact-cached analysis files are consistent across systems.
# Note that we have one global, well-known symlink dir, again so that paths are
# consistent across builds.
symlink_dir = os.path.join(ivy_workdir, 'jars')
# Note that it's possible for all targets to be valid but for no classpath file to exist at
# target_classpath_file, e.g., if we previously built a superset of targets.
if report_missing or invalidation_check.invalid_vts or not os.path.exists(
raw_target_classpath_file):
args = ['-cachepath', raw_target_classpath_file_tmp
] + (custom_args if custom_args else [])
self.exec_ivy(
target_workdir=target_workdir,
targets=global_vts.targets,
args=args,
executor=executor,
ivy=ivy,
workunit_name=workunit_name,
confs=confs,
use_soft_excludes=self.get_options().soft_excludes,
resolve_hash_name=resolve_hash_name)
if not os.path.exists(raw_target_classpath_file_tmp):
raise TaskError(
'Ivy failed to create classpath file at {}'.format(
raw_target_classpath_file_tmp))
shutil.move(raw_target_classpath_file_tmp,
raw_target_classpath_file)
logger.debug('Moved ivy classfile file to {dest}'.format(
dest=raw_target_classpath_file))
if self.artifact_cache_writes_enabled():
self.update_artifact_cache([(global_vts,
[raw_target_classpath_file])])
else:
logger.debug("Using previously resolved reports: {}".format(
report_paths))
# Make our actual classpath be symlinks, so that the paths are uniform across systems.
# Note that we must do this even if we read the raw_target_classpath_file from the artifact
# cache. If we cache the target_classpath_file we won't know how to create the symlinks.
with IvyTaskMixin.symlink_map_lock:
products = self.context.products
existing_symlinks_map = products.get_data(
'ivy_resolve_symlink_map', lambda: dict())
symlink_map = IvyUtils.symlink_cachepath(
IvySubsystem.global_instance().get_options().cache_dir,
raw_target_classpath_file, symlink_dir, target_classpath_file,
existing_symlinks_map)
existing_symlinks_map.update(symlink_map)
with IvyUtils.cachepath(target_classpath_file) as classpath:
stripped_classpath = [path.strip() for path in classpath]
return (stripped_classpath, resolve_hash_name)
def ivy_resolve(self,
targets,
executor=None,
silent=False,
workunit_name=None,
confs=None,
custom_args=None):
"""Populates the product 'ivy_resolve_symlink_map' from the specified targets."""
if not targets:
return ([], set())
# NOTE: Always pass all the targets to exec_ivy, as they're used to calculate the name of
# the generated module, which in turn determines the location of the XML report file
# ivy generates. We recompute this name from targets later in order to find that file.
# TODO: This is fragile. Refactor so that we're not computing the name twice.
ivy = Bootstrapper.default_ivy(bootstrap_workunit_factory=self.context.new_workunit)
ivy_workdir = os.path.join(self.context.options.for_global_scope().pants_workdir, 'ivy')
fingerprint_strategy = IvyResolveFingerprintStrategy()
with self.invalidated(targets,
invalidate_dependents=False,
silent=silent,
fingerprint_strategy=fingerprint_strategy) as invalidation_check:
if not invalidation_check.all_vts:
return ([], set())
global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)
target_workdir = os.path.join(ivy_workdir, global_vts.cache_key.hash)
target_classpath_file = os.path.join(target_workdir, 'classpath')
raw_target_classpath_file = target_classpath_file + '.raw'
raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp'
# A common dir for symlinks into the ivy2 cache. This ensures that paths to jars
# in artifact-cached analysis files are consistent across systems.
# Note that we have one global, well-known symlink dir, again so that paths are
# consistent across builds.
symlink_dir = os.path.join(ivy_workdir, 'jars')
# Note that it's possible for all targets to be valid but for no classpath file to exist at
# target_classpath_file, e.g., if we previously built a superset of targets.
if invalidation_check.invalid_vts or not os.path.exists(raw_target_classpath_file):
args = ['-cachepath', raw_target_classpath_file_tmp] + (custom_args if custom_args else [])
self.exec_ivy(
target_workdir=target_workdir,
targets=global_vts.targets,
args=args,
executor=executor,
ivy=ivy,
workunit_name=workunit_name,
confs=confs)
if not os.path.exists(raw_target_classpath_file_tmp):
raise TaskError('Ivy failed to create classpath file at %s'
% raw_target_classpath_file_tmp)
shutil.move(raw_target_classpath_file_tmp, raw_target_classpath_file)
logger.debug('Copied ivy classfile file to {dest}'.format(dest=raw_target_classpath_file))
if self.artifact_cache_writes_enabled():
self.update_artifact_cache([(global_vts, [raw_target_classpath_file])])
# Make our actual classpath be symlinks, so that the paths are uniform across systems.
# Note that we must do this even if we read the raw_target_classpath_file from the artifact
# cache. If we cache the target_classpath_file we won't know how to create the symlinks.
symlink_map = IvyUtils.symlink_cachepath(ivy.ivy_cache_dir, raw_target_classpath_file,
symlink_dir, target_classpath_file)
with IvyTaskMixin.symlink_map_lock:
products = self.context.products
all_symlinks_map = products.get_data('ivy_resolve_symlink_map') or defaultdict(list)
for path, symlink in symlink_map.items():
all_symlinks_map[os.path.realpath(path)].append(symlink)
products.safe_create_data('ivy_resolve_symlink_map',
lambda: all_symlinks_map)
with IvyUtils.cachepath(target_classpath_file) as classpath:
stripped_classpath = [path.strip() for path in classpath]
return (stripped_classpath, global_vts.targets)
