def combine_versioned_targets(vts):
targets = []
for vt in vts:
targets.extend(vt.targets)
cache_key = CacheKeyGenerator.combine_cache_keys([vt.cache_key for vt in vts])
valid = all([vt.valid for vt in vts])
return VersionedTargetSet(targets, cache_key, valid)
def ivy_resolve(self, targets, java_runner=None, symlink_ivyxml=False, silent=False,
workunit_name=None, workunit_labels=None):
java_runner = java_runner or runjava_indivisible
targets = set(targets)
if not targets:
return []
work_dir = self.context.config.get('ivy-resolve', 'workdir')
confs = self.context.config.getlist('ivy-resolve', 'confs')
with self.invalidated(targets,
only_buildfiles=True,
invalidate_dependents=True,
silent=silent) as invalidation_check:
global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)
target_workdir = os.path.join(work_dir, global_vts.cache_key.hash)
target_classpath_file = os.path.join(target_workdir, 'classpath')
raw_target_classpath_file = target_classpath_file + '.raw'
raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp'
symlink_dir = os.path.join(target_workdir, 'jars')
# Note that it's possible for all targets to be valid but for no classpath file to exist at
# target_classpath_file, e.g., if we previously built a superset of targets.
if invalidation_check.invalid_vts or not os.path.exists(raw_target_classpath_file):
ivy_utils = IvyUtils(config=self.context.config,
options=self.context.options,
log=self.context.log)
args = (['-cachepath', raw_target_classpath_file_tmp] +
['-confs'] + confs)
def exec_ivy():
ivy_utils.exec_ivy(
target_workdir=target_workdir,
targets=targets,
args=args,
runjava=java_runner,
workunit_name='ivy',
workunit_factory=self.context.new_workunit,
symlink_ivyxml=symlink_ivyxml,
)
if workunit_name:
with self.context.new_workunit(name=workunit_name, labels=workunit_labels or []):
exec_ivy()
else:
exec_ivy()
if not os.path.exists(raw_target_classpath_file_tmp):
raise TaskError('Ivy failed to create classpath file at %s' % raw_target_classpath_file_tmp)
shutil.move(raw_target_classpath_file_tmp, raw_target_classpath_file)
if self.artifact_cache_writes_enabled():
self.update_artifact_cache([(global_vts, [raw_target_classpath_file])])
# Make our actual classpath be symlinks, so that the paths are uniform across systems.
# Note that we must do this even if we read the raw_target_classpath_file from the artifact
# cache. If we cache the target_classpath_file we won't know how to create the symlinks.
symlink_map = IvyUtils.symlink_cachepath(raw_target_classpath_file, symlink_dir, target_classpath_file)
with Task.symlink_map_lock:
all_symlinks_map = self.context.products.get_data('symlink_map') or defaultdict(list)
for path, symlink in symlink_map.items():
all_symlinks_map[os.path.realpath(path)].append(symlink)
self.context.products.set_data('symlink_map', all_symlinks_map)
with IvyUtils.cachepath(target_classpath_file) as classpath:
stripped_classpath = [path.strip() for path in classpath]
return [path for path in stripped_classpath if IvyUtils.is_mappable_artifact(path)]
def check_artifact_cache_for(self, invalidation_check): # Ivy resolution is an output dependent on the entire target set, and is not divisible # by target. So we can only cache it keyed by the entire target set. global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts) return [global_vts]
def check_artifact_cache_for(self, invalidation_check):
# Ivy resolution is an output dependent on the entire target set, and is not divisible
# by target. So we can only cache it keyed by the entire target set.
global_vts = VersionedTargetSet.from_versioned_targets(
invalidation_check.all_vts)
return [global_vts]
def execute(self, targets):
"""Resolves the specified confs for the configured targets and returns an iterator over
tuples of (conf, jar path).
"""
def dirname_for_requested_targets(targets):
"""Where we put the classpath file for this set of targets."""
sha = hashlib.sha1()
for t in targets:
sha.update(t.id)
return sha.hexdigest()
def is_classpath(target):
return is_jar(target) or (is_internal(target) and any(
jar for jar in target.jar_dependencies if jar.rev))
groups = self.context.products.get_data('exclusives_groups')
# Below, need to take the code that actually execs ivy, and invoke it once for each
# group. Then after running ivy, we need to take the resulting classpath, and load it into
# the build products.
# The set of groups we need to consider is complicated:
# - If there are no conflicting exclusives (ie, there's only one entry in the map),
# then we just do the one.
# - If there are conflicts, then there will be at least three entries in the groups map:
# - the group with no exclusives (X)
# - the two groups that are in conflict (A and B).
# In the latter case, we need to do the resolve twice: Once for A+X, and once for B+X,
# because things in A and B can depend on things in X; and so they can indirectly depend
# on the dependencies of X. (I think this well be covered by the computed transitive dependencies of
# A and B. But before pushing this change, review this comment, and make sure that this is
# working correctly.
for group_key in groups.get_group_keys():
# Narrow the groups target set to just the set of targets that we're supposed to build.
# Normally, this shouldn't be different from the contents of the group.
group_targets = groups.get_targets_for_group_key(group_key) & set(
targets)
classpath_targets = OrderedSet()
for target in group_targets:
classpath_targets.update(
filter(is_classpath, filter(is_concrete,
target.resolve())))
if len(classpath_targets) == 0:
continue # Nothing to do.
target_workdir = os.path.join(
self._work_dir, dirname_for_requested_targets(group_targets))
target_classpath_file = os.path.join(target_workdir, 'classpath')
with self.invalidated(
classpath_targets,
only_buildfiles=True,
invalidate_dependents=True) as invalidation_check:
# Note that it's possible for all targets to be valid but for no classpath file to exist at
# target_classpath_file, e.g., if we previously built a superset of targets.
if invalidation_check.invalid_vts or not os.path.exists(
target_classpath_file):
# TODO(benjy): s/targets/classpath_targets/ ??
self._exec_ivy(
target_workdir, targets,
['-cachepath', target_classpath_file, '-confs'] +
self._confs)
if not os.path.exists(target_classpath_file):
raise TaskError(
'Ivy failed to create classpath file at %s %s' %
target_classpath_file)
if self.get_artifact_cache(
) and self.context.options.write_to_artifact_cache:
global_vts = VersionedTargetSet.from_versioned_targets(
invalidation_check.all_vts)
self.update_artifact_cache([(global_vts,
[target_classpath_file])])
with self._cachepath(target_classpath_file) as classpath:
for path in classpath:
if self._map_jar(path):
for conf in self._confs:
groups.update_compatible_classpaths(
group_key, [(conf, path.strip())])
if self._report:
self._generate_ivy_report()
if self.context.products.isrequired("ivy_jar_products"):
self._populate_ivy_jar_products()
create_jardeps_for = self.context.products.isrequired(
self._mapfor_typename())
if create_jardeps_for:
genmap = self.context.products.get(self._mapfor_typename())
for target in filter(create_jardeps_for, targets):
self._mapjars(genmap, target)
def ivy_resolve(self,
targets,
java_runner=None,
ivy_args=None,
symlink_ivyxml=False,
silent=False,
workunit_name=None,
workunit_labels=None):
java_runner = java_runner or runjava_indivisible
ivy_args = ivy_args or []
targets = set(targets)
if not targets:
return []
work_dir = self.context.config.get('ivy-resolve', 'workdir')
confs = self.context.config.getlist('ivy-resolve', 'confs')
with self.invalidated(targets,
only_buildfiles=True,
invalidate_dependents=True,
silent=silent) as invalidation_check:
global_vts = VersionedTargetSet.from_versioned_targets(
invalidation_check.all_vts)
target_workdir = os.path.join(work_dir, global_vts.cache_key.hash)
target_classpath_file = os.path.join(target_workdir, 'classpath')
target_classpath_file_tmp = target_classpath_file + '.tmp'
# Note that it's possible for all targets to be valid but for no classpath file to exist at
# target_classpath_file, e.g., if we previously built a superset of targets.
if invalidation_check.invalid_vts or not os.path.exists(
target_classpath_file):
ivy_utils = IvyUtils(config=self.context.config,
options=self.context.options,
log=self.context.log)
args = (['-cachepath', target_classpath_file_tmp] +
['-confs'] + confs + ivy_args)
def exec_ivy():
ivy_utils.exec_ivy(
target_workdir=target_workdir,
targets=targets,
args=args,
runjava=java_runner,
workunit_name='ivy',
workunit_factory=self.context.new_workunit,
symlink_ivyxml=symlink_ivyxml,
)
if workunit_name:
with self.context.new_workunit(name=workunit_name,
labels=workunit_labels
or []):
exec_ivy()
else:
exec_ivy()
if not os.path.exists(target_classpath_file_tmp):
raise TaskError(
'Ivy failed to create classpath file at %s' %
target_classpath_file_tmp)
shutil.move(target_classpath_file_tmp, target_classpath_file)
if self.get_artifact_cache(
) and self.context.options.write_to_artifact_cache:
self.update_artifact_cache([(global_vts,
[target_classpath_file])])
with IvyUtils.cachepath(target_classpath_file) as classpath:
stripped_classpath = [path.strip() for path in classpath]
return [
path for path in stripped_classpath
if IvyUtils.is_mappable_artifact(path)
]
def execute(self, targets):
"""Resolves the specified confs for the configured targets and returns an iterator over
tuples of (conf, jar path).
"""
def dirname_for_requested_targets(targets):
"""Where we put the classpath file for this set of targets."""
sha = hashlib.sha1()
for t in targets:
sha.update(t.id)
return sha.hexdigest()
def is_classpath(target):
return is_jar(target) or (
is_internal(target) and any(jar for jar in target.jar_dependencies if jar.rev)
)
groups = self.context.products.get_data('exclusives_groups')
# Below, need to take the code that actually execs ivy, and invoke it once for each
# group. Then after running ivy, we need to take the resulting classpath, and load it into
# the build products.
# The set of groups we need to consider is complicated:
# - If there are no conflicting exclusives (ie, there's only one entry in the map),
# then we just do the one.
# - If there are conflicts, then there will be at least three entries in the groups map:
# - the group with no exclusives (X)
# - the two groups that are in conflict (A and B).
# In the latter case, we need to do the resolve twice: Once for A+X, and once for B+X,
# because things in A and B can depend on things in X; and so they can indirectly depend
# on the dependencies of X. (I think this well be covered by the computed transitive dependencies of
# A and B. But before pushing this change, review this comment, and make sure that this is
# working correctly.
for group_key in groups.get_group_keys():
# Narrow the groups target set to just the set of targets that we're supposed to build.
# Normally, this shouldn't be different from the contents of the group.
group_targets = groups.get_targets_for_group_key(group_key) & set(targets)
classpath_targets = OrderedSet()
for target in group_targets:
classpath_targets.update(filter(is_classpath, filter(is_concrete, target.resolve())))
if len(classpath_targets) == 0:
continue # Nothing to do.
target_workdir = os.path.join(self._work_dir, dirname_for_requested_targets(group_targets))
target_classpath_file = os.path.join(target_workdir, 'classpath')
with self.invalidated(classpath_targets, only_buildfiles=True,
invalidate_dependents=True) as invalidation_check:
# Note that it's possible for all targets to be valid but for no classpath file to exist at
# target_classpath_file, e.g., if we previously built a superset of targets.
if invalidation_check.invalid_vts or not os.path.exists(target_classpath_file):
# TODO(benjy): s/targets/classpath_targets/ ??
self._exec_ivy(target_workdir, targets, [
'-cachepath', target_classpath_file,
'-confs'
] + self._confs)
if not os.path.exists(target_classpath_file):
raise TaskError('Ivy failed to create classpath file at %s %s' % target_classpath_file)
if self.get_artifact_cache() and self.context.options.write_to_artifact_cache:
global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)
self.update_artifact_cache([(global_vts, [target_classpath_file])])
with self._cachepath(target_classpath_file) as classpath:
for path in classpath:
if self._map_jar(path):
for conf in self._confs:
groups.update_compatible_classpaths(group_key, [(conf, path.strip())])
if self._report:
self._generate_ivy_report()
if self.context.products.isrequired("ivy_jar_products"):
self._populate_ivy_jar_products()
create_jardeps_for = self.context.products.isrequired(self._mapfor_typename())
if create_jardeps_for:
genmap = self.context.products.get(self._mapfor_typename())
for target in filter(create_jardeps_for, targets):
self._mapjars(genmap, target)