def _merge_artifact(self, versioned_target_set):
"""Merges artifacts representing the individual targets in a VersionedTargetSet into one artifact for that set.
Creates an output classes dir, depfile and analysis file for the VersionedTargetSet.
Note that the merged artifact may be incomplete (e.g., if we have no previous artifacts for some of the
individual targets). That's OK: We run this right before we invoke zinc, which will fill in what's missing.
This method is not required for correctness, only for efficiency: it can prevent zinc from doing superfluous work.
NOTE: This method is reentrant.
"""
if len(versioned_target_set.targets) <= 1:
return # Nothing to do.
with temporary_dir() as tmpdir:
dst_classes_dir, dst_depfile, dst_analysis_file = self._output_paths(versioned_target_set.targets)
safe_rmtree(dst_classes_dir)
safe_mkdir(dst_classes_dir)
src_analysis_files = []
# TODO: Do we actually need to merge deps? Zinc will stomp them anyway on success.
dst_deps = Dependencies(dst_classes_dir)
for target in versioned_target_set.targets:
src_classes_dir, src_depfile, src_analysis_file = self._output_paths([target])
if os.path.exists(src_depfile):
src_deps = Dependencies(src_classes_dir)
src_deps.load(src_depfile)
dst_deps.merge(src_deps)
classes_by_source = src_deps.findclasses([target]).get(target, {})
for source, classes in classes_by_source.items():
for cls in classes:
src = os.path.join(src_classes_dir, cls)
dst = os.path.join(dst_classes_dir, cls)
# src may not exist if we aborted a build in the middle. That's OK: zinc will notice that
# it's missing and rebuild it.
# dst may already exist if we have overlapping targets. It's not a good idea
# to have those, but until we enforce it, we must allow it here.
if os.path.exists(src) and not os.path.exists(dst):
# Copy the class file.
safe_mkdir(os.path.dirname(dst))
os.link(src, dst)
# Rebase a copy of the per-target analysis files to reflect the merged classes dir.
if os.path.exists(src_analysis_file):
src_analysis_file_tmp = \
os.path.join(tmpdir, os.path.relpath(src_analysis_file, self._analysis_files_base))
shutil.copyfile(src_analysis_file, src_analysis_file_tmp)
src_analysis_files.append(src_analysis_file_tmp)
if self._zinc_utils.run_zinc_rebase(src_analysis_file_tmp, [(src_classes_dir, dst_classes_dir)]):
self.context.log.warn('In merge_artifact: zinc failed to rebase analysis file %s. '\
'Target may require a full rebuild.' %\
src_analysis_file_tmp)
dst_deps.save(dst_depfile)
if self._zinc_utils.run_zinc_merge(src_analysis_files, dst_analysis_file):
self.context.log.warn('zinc failed to merge analysis files %s to %s. '\
'Target may require a full rebuild.' %\
(':'.join(src_analysis_files), dst_analysis_file))
def split_artifact(self, deps, versioned_target_set):
if len(versioned_target_set.targets) <= 1:
return
buildroot = get_buildroot()
classes_by_source_by_target = deps.findclasses(versioned_target_set.targets)
src_output_dir, _, src_analysis_cache = self.create_output_paths(versioned_target_set.targets)
analysis_splits = [] # List of triples of (list of sources, destination output dir, destination analysis cache).
# for dependency analysis, we need to record the cache files that we create in the split
for target in versioned_target_set.targets:
classes_by_source = classes_by_source_by_target.get(target, {})
dst_output_dir, dst_depfile, dst_analysis_cache = self.create_output_paths([target])
safe_rmtree(dst_output_dir)
safe_mkdir(dst_output_dir)
sources = []
dst_deps = Dependencies(dst_output_dir)
for source, classes in classes_by_source.items():
src = os.path.join(target.target_base, source)
dst_deps.add(src, classes)
source_abspath = os.path.join(buildroot, target.target_base, source)
sources.append(source_abspath)
for cls in classes:
# Copy the class file.
dst = os.path.join(dst_output_dir, cls)
safe_mkdir(os.path.dirname(dst))
os.link(os.path.join(src_output_dir, cls), dst)
dst_deps.save(dst_depfile)
analysis_splits.append((sources, dst_output_dir, dst_analysis_cache))
self.generated_caches.add(os.path.join(dst_output_dir, dst_analysis_cache))
# Use zinc to split the analysis files.
if os.path.exists(src_analysis_cache):
analysis_args = []
analysis_args.extend(self._zinc_jar_args)
analysis_args.extend([
'-log-level', self.context.options.log_level or 'info',
'-analysis',
'-mirror-analysis'
])
split_args = analysis_args + [
'-cache', src_analysis_cache,
'-split', ','.join(['{%s}:%s' % (':'.join(x[0]), x[2]) for x in analysis_splits]),
]
if self.runjava(self._main, classpath=self._zinc_classpath, args=split_args, jvmargs=self._jvm_args):
raise TaskError, 'zinc failed to split analysis files %s from %s' %\
(':'.join([x[2] for x in analysis_splits]), src_analysis_cache)
# Now rebase the newly created analysis files.
for split in analysis_splits:
dst_analysis_cache = split[2]
if os.path.exists(dst_analysis_cache):
rebase_args = analysis_args + [
'-cache', dst_analysis_cache,
'-rebase', '%s:%s' % (src_output_dir, split[1]),
]
if self.runjava(self._main, classpath=self._zinc_classpath, args=rebase_args, jvmargs=self._jvm_args):
raise TaskError, 'In split_artifact: zinc failed to rebase analysis file %s' % dst_analysis_cache
def _split_artifact(self, deps, versioned_target_set):
"""Splits an artifact representing several targets into target-by-target artifacts.
Creates an output classes dir, a depfile and an analysis file for each target.
Note that it's not OK to create incomplete artifacts here: this is run *after* a zinc invocation,
and the expectation is that the result is complete.
NOTE: This method is reentrant.
"""
if len(versioned_target_set.targets) <= 1:
return
classes_by_source_by_target = deps.findclasses(versioned_target_set.targets)
src_classes_dir, _, src_analysis_file = self._output_paths(versioned_target_set.targets)
# Specifies that the list of sources defines a split to the classes dir and analysis file.
SplitInfo = namedtuple('SplitInfo', ['sources', 'dst_classes_dir', 'dst_analysis_file'])
analysis_splits = [] # List of SplitInfos.
portable_analysis_splits = [] # The same, for the portable version of the analysis cache.
# Prepare the split arguments.
for target in versioned_target_set.targets:
classes_by_source = classes_by_source_by_target.get(target, {})
dst_classes_dir, dst_depfile, dst_analysis_file = self._output_paths([target])
safe_rmtree(dst_classes_dir)
safe_mkdir(dst_classes_dir)
sources = []
dst_deps = Dependencies(dst_classes_dir)
for source, classes in classes_by_source.items():
src = os.path.join(target.target_base, source)
dst_deps.add(src, classes)
sources.append(os.path.join(target.target_base, source))
for cls in classes:
# Copy the class file.
dst = os.path.join(dst_classes_dir, cls)
safe_mkdir(os.path.dirname(dst))
os.link(os.path.join(src_classes_dir, cls), dst)
dst_deps.save(dst_depfile)
analysis_splits.append(SplitInfo(sources, dst_classes_dir, dst_analysis_file))
portable_analysis_splits.append(SplitInfo(sources, dst_classes_dir, _portable(dst_analysis_file)))
def do_split(src_analysis_file, splits):
if os.path.exists(src_analysis_file):
if self._zinc_utils.run_zinc_split(src_analysis_file, [(x.sources, x.dst_analysis_file) for x in splits]):
raise TaskError, 'zinc failed to split analysis files %s from %s' %\
(':'.join([x.dst_analysis_file for x in splits]), src_analysis_file)
for split in splits:
if os.path.exists(split.dst_analysis_file):
if self._zinc_utils.run_zinc_rebase(split.dst_analysis_file,
[(src_classes_dir, split.dst_classes_dir)]):
raise TaskError, \
'In split_artifact: zinc failed to rebase analysis file %s' % split.dst_analysis_file
# Now rebase the newly created analysis file(s) to reflect the split classes dirs.
do_split(src_analysis_file, analysis_splits)
do_split(_portable(src_analysis_file), portable_analysis_splits)
def split_depfile(self, deps, versioned_target_set):
if len(versioned_target_set.targets) <= 1:
return
classes_by_source_by_target = deps.findclasses(versioned_target_set.targets)
for target in versioned_target_set.targets:
classes_by_source = classes_by_source_by_target.get(target, {})
dst_depfile = self.create_depfile_path([target])
dst_deps = Dependencies(self._classes_dir)
for source, classes in classes_by_source.items():
src = os.path.join(target.target_base, source)
dst_deps.add(src, classes)
dst_deps.save(dst_depfile)
def split_depfile(self, deps, versioned_target_set):
if len(versioned_target_set.targets) <= 1:
return
classes_by_source_by_target = deps.findclasses(
versioned_target_set.targets)
for target in versioned_target_set.targets:
classes_by_source = classes_by_source_by_target.get(target, {})
dst_depfile = self.create_depfile_path([target])
dst_deps = Dependencies(self._classes_dir)
for source, classes in classes_by_source.items():
src = os.path.join(target.target_base, source)
dst_deps.add(src, classes)
dst_deps.save(dst_depfile)
def merge_artifact(self, versioned_target_set):
if len(versioned_target_set.targets) <= 1:
return
with temporary_dir() as tmpdir:
dst_output_dir, dst_depfile, dst_analysis_cache = self.create_output_paths(versioned_target_set.targets)
safe_rmtree(dst_output_dir)
safe_mkdir(dst_output_dir)
src_analysis_caches = []
# TODO: Do we actually need to merge deps? Zinc will stomp them anyway on success.
dst_deps = Dependencies(dst_output_dir)
for target in versioned_target_set.targets:
src_output_dir, src_depfile, src_analysis_cache = self.create_output_paths([target])
if os.path.exists(src_depfile):
src_deps = Dependencies(src_output_dir)
src_deps.load(src_depfile)
dst_deps.merge(src_deps)
classes_by_source = src_deps.findclasses([target]).get(target, {})
for source, classes in classes_by_source.items():
for cls in classes:
src = os.path.join(src_output_dir, cls)
dst = os.path.join(dst_output_dir, cls)
# src may not exist if we aborted a build in the middle. That's OK: zinc will notice that
# it's missing and rebuild it.
# dst may already exist if we have overlapping targets. It's not a good idea
# to have those, but until we enforce it, we must allow it here.
if os.path.exists(src) and not os.path.exists(dst):
# Copy the class file.
safe_mkdir(os.path.dirname(dst))
os.link(src, dst)
# Rebase a copy of the per-target analysis files prior to merging.
if os.path.exists(src_analysis_cache):
src_analysis_cache_tmp = \
os.path.join(tmpdir, os.path.relpath(src_analysis_cache, self._analysis_cache_dir))
shutil.copyfile(src_analysis_cache, src_analysis_cache_tmp)
src_analysis_caches.append(src_analysis_cache_tmp)
if self._zinc_utils.run_zinc_rebase(cache=src_analysis_cache_tmp, rebasings=[(src_output_dir, dst_output_dir)]):
self.context.log.warn('In merge_artifact: zinc failed to rebase analysis file %s. ' \
'Target may require a full rebuild.' % \
src_analysis_cache_tmp)
dst_deps.save(dst_depfile)
if self._zinc_utils.run_zinc_merge(src_caches=src_analysis_caches, dst_cache=dst_analysis_cache):
self.context.log.warn('zinc failed to merge analysis files %s to %s. ' \
'Target may require a full rebuild.' % \
(':'.join(src_analysis_caches), dst_analysis_cache))
def merge_depfile(self, versioned_target_set):
if len(versioned_target_set.targets) <= 1:
return
dst_depfile = self.create_depfile_path(versioned_target_set.targets)
dst_deps = Dependencies(self._classes_dir)
for target in versioned_target_set.targets:
src_depfile = self.create_depfile_path([target])
if os.path.exists(src_depfile):
src_deps = Dependencies(self._classes_dir)
src_deps.load(src_depfile)
dst_deps.merge(src_deps)
dst_deps.save(dst_depfile)
def post_process_cached_vts(cached_vts):
# Merge the cached analyses into the existing global one.
if cached_vts:
with self.context.new_workunit(name='merge-dependencies'):
global_deps = Dependencies(self._classes_dir)
if os.path.exists(self._depfile):
global_deps.load(self._depfile)
for vt in cached_vts:
for target in vt.targets:
depfile = JavaCompile.create_depfile_path(self._depfile_tmpdir, [target])
if os.path.exists(depfile):
deps = Dependencies(self._classes_dir)
deps.load(depfile)
global_deps.merge(deps)
global_deps.save(self._depfile)
def split_depfile(self, vt):
depfile = self.create_depfile_path(vt.targets)
if len(vt.targets) <= 1 or not os.path.exists(depfile) or self.dry_run:
return
deps = Dependencies(self._classes_dir)
deps.load(depfile)
classes_by_source_by_target = deps.findclasses(vt.targets)
for target in vt.targets:
classes_by_source = classes_by_source_by_target.get(target, {})
dst_depfile = self.create_depfile_path([target])
dst_deps = Dependencies(self._classes_dir)
for source, classes in classes_by_source.items():
src = os.path.join(target.target_base, source)
dst_deps.add(src, classes)
dst_deps.save(dst_depfile)
def split_artifact(self, deps, versioned_target_set):
if len(versioned_target_set.targets) <= 1:
return
classes_by_source_by_target = deps.findclasses(versioned_target_set.targets)
src_output_dir, _, src_analysis_cache = self.create_output_paths(versioned_target_set.targets)
analysis_splits = [] # List of triples of (list of sources, destination output dir, destination analysis cache).
# for dependency analysis, we need to record the cache files that we create in the split
for target in versioned_target_set.targets:
classes_by_source = classes_by_source_by_target.get(target, {})
dst_output_dir, dst_depfile, dst_analysis_cache = self.create_output_paths([target])
safe_rmtree(dst_output_dir)
safe_mkdir(dst_output_dir)
sources = []
dst_deps = Dependencies(dst_output_dir)
for source, classes in classes_by_source.items():
src = os.path.join(target.target_base, source)
dst_deps.add(src, classes)
sources.append(os.path.join(target.target_base, source))
for cls in classes:
# Copy the class file.
dst = os.path.join(dst_output_dir, cls)
safe_mkdir(os.path.dirname(dst))
os.link(os.path.join(src_output_dir, cls), dst)
dst_deps.save(dst_depfile)
analysis_splits.append((sources, dst_output_dir, dst_analysis_cache))
self.generated_caches.add(os.path.join(dst_output_dir, dst_analysis_cache))
# Split the analysis files.
if os.path.exists(src_analysis_cache):
if self._zinc_utils.run_zinc_split(src_cache=src_analysis_cache, splits=[(x[0], x[2]) for x in analysis_splits]):
raise TaskError, 'zinc failed to split analysis files %s from %s' %\
(':'.join([x[2] for x in analysis_splits]), src_analysis_cache)
# Now rebase the newly created analysis files.
for split in analysis_splits:
dst_analysis_cache = split[2]
dst_output_dir = split[1]
if os.path.exists(dst_analysis_cache):
rebasings = [(src_output_dir, dst_output_dir)]
if self._zinc_utils.run_zinc_rebase(cache=dst_analysis_cache, rebasings=rebasings):
raise TaskError, 'In split_artifact: zinc failed to rebase analysis file %s' % dst_analysis_cache
def merge_artifact(self, versioned_target_set):
if len(versioned_target_set.targets) <= 1:
return
with temporary_dir() as tmpdir:
dst_output_dir, dst_depfile, dst_analysis_cache = self.create_output_paths(versioned_target_set.targets)
safe_rmtree(dst_output_dir)
safe_mkdir(dst_output_dir)
src_analysis_caches = []
analysis_args = []
analysis_args.extend(self._zinc_jar_args)
analysis_args.extend([
'-log-level', self.context.options.log_level or 'info',
'-analysis',
])
# TODO: Do we actually need to merge deps? Zinc will stomp them anyway on success.
dst_deps = Dependencies(dst_output_dir)
for target in versioned_target_set.targets:
src_output_dir, src_depfile, src_analysis_cache = self.create_output_paths([target])
if os.path.exists(src_depfile):
src_deps = Dependencies(src_output_dir)
src_deps.load(src_depfile)
dst_deps.merge(src_deps)
classes_by_source = src_deps.findclasses([target]).get(target, {})
for source, classes in classes_by_source.items():
for cls in classes:
src = os.path.join(src_output_dir, cls)
dst = os.path.join(dst_output_dir, cls)
# src may not exist if we aborted a build in the middle. That's OK: zinc will notice that
# it's missing and rebuild it.
# dst may already exist if we have overlapping targets. It's not a good idea
# to have those, but until we enforce it, we must allow it here.
if os.path.exists(src) and not os.path.exists(dst):
# Copy the class file.
safe_mkdir(os.path.dirname(dst))
os.link(src, dst)
# Use zinc to rebase a copy of the per-target analysis files prior to merging.
if os.path.exists(src_analysis_cache):
src_analysis_cache_tmp = \
os.path.join(tmpdir, os.path.relpath(src_analysis_cache, self._analysis_cache_dir))
shutil.copyfile(src_analysis_cache, src_analysis_cache_tmp)
src_analysis_caches.append(src_analysis_cache_tmp)
rebase_args = analysis_args + [
'-cache', src_analysis_cache_tmp,
'-rebase', '%s:%s' % (src_output_dir, dst_output_dir),
]
if self.runjava(self._main, classpath=self._zinc_classpath, args=rebase_args, jvmargs=self._jvm_args):
self.context.log.warn('In merge_artifact: zinc failed to rebase analysis file %s. ' \
'Target may require a full rebuild.' % src_analysis_cache_tmp)
dst_deps.save(dst_depfile)
# Use zinc to merge the analysis files.
merge_args = analysis_args + [
'-cache', dst_analysis_cache,
'-merge', ':'.join(src_analysis_caches),
]
if self.runjava(self._main, classpath=self._zinc_classpath, args=merge_args, jvmargs=self._jvm_args):
raise TaskError, 'zinc failed to merge analysis files %s to %s' % \
(':'.join(src_analysis_caches), dst_analysis_cache)
def merge_artifact(self, versioned_target_set):
if len(versioned_target_set.targets) <= 1:
return
with temporary_dir() as tmpdir:
dst_output_dir, dst_depfile, dst_analysis_cache = self.create_output_paths(
versioned_target_set.targets)
safe_rmtree(dst_output_dir)
safe_mkdir(dst_output_dir)
src_analysis_caches = []
analysis_args = []
analysis_args.extend(self._zinc_jar_args)
analysis_args.extend([
'-log-level',
self.context.options.log_level or 'info',
'-analysis',
])
# TODO: Do we actually need to merge deps? Zinc will stomp them anyway on success.
dst_deps = Dependencies(dst_output_dir)
for target in versioned_target_set.targets:
src_output_dir, src_depfile, src_analysis_cache = self.create_output_paths(
[target])
if os.path.exists(src_depfile):
src_deps = Dependencies(src_output_dir)
src_deps.load(src_depfile)
dst_deps.merge(src_deps)
classes_by_source = src_deps.findclasses([target]).get(
target, {})
for source, classes in classes_by_source.items():
for cls in classes:
src = os.path.join(src_output_dir, cls)
dst = os.path.join(dst_output_dir, cls)
# src may not exist if we aborted a build in the middle. That's OK: zinc will notice that
# it's missing and rebuild it.
# dst may already exist if we have overlapping targets. It's not a good idea
# to have those, but until we enforce it, we must allow it here.
if os.path.exists(src) and not os.path.exists(dst):
# Copy the class file.
safe_mkdir(os.path.dirname(dst))
os.link(src, dst)
# Use zinc to rebase a copy of the per-target analysis files prior to merging.
if os.path.exists(src_analysis_cache):
src_analysis_cache_tmp = \
os.path.join(tmpdir, os.path.relpath(src_analysis_cache, self._analysis_cache_dir))
shutil.copyfile(src_analysis_cache,
src_analysis_cache_tmp)
src_analysis_caches.append(src_analysis_cache_tmp)
rebase_args = analysis_args + [
'-cache',
src_analysis_cache_tmp,
'-rebase',
'%s:%s' % (src_output_dir, dst_output_dir),
]
if self.runjava(self._main,
classpath=self._zinc_classpath,
args=rebase_args,
jvmargs=self._jvm_args):
self.context.log.warn('In merge_artifact: zinc failed to rebase analysis file %s. ' \
'Target may require a full rebuild.' % src_analysis_cache_tmp)
dst_deps.save(dst_depfile)
# Use zinc to merge the analysis files.
merge_args = analysis_args + [
'-cache',
dst_analysis_cache,
'-merge',
':'.join(src_analysis_caches),
]
if self.runjava(self._main,
classpath=self._zinc_classpath,
args=merge_args,
jvmargs=self._jvm_args):
raise TaskError, 'zinc failed to merge analysis files %s to %s' % \
(':'.join(src_analysis_caches), dst_analysis_cache)
def split_artifact(self, deps, versioned_target_set):
if len(versioned_target_set.targets) <= 1:
return
buildroot = get_buildroot()
classes_by_source_by_target = deps.findclasses(
versioned_target_set.targets)
src_output_dir, _, src_analysis_cache = self.create_output_paths(
versioned_target_set.targets)
analysis_splits = [
] # List of triples of (list of sources, destination output dir, destination analysis cache).
for target in versioned_target_set.targets:
classes_by_source = classes_by_source_by_target.get(target, {})
dst_output_dir, dst_depfile, dst_analysis_cache = self.create_output_paths(
[target])
safe_rmtree(dst_output_dir)
safe_mkdir(dst_output_dir)
sources = []
dst_deps = Dependencies(dst_output_dir)
for source, classes in classes_by_source.items():
src = os.path.join(target.target_base, source)
dst_deps.add(src, classes)
source_abspath = os.path.join(buildroot, target.target_base,
source)
sources.append(source_abspath)
for cls in classes:
# Copy the class file.
dst = os.path.join(dst_output_dir, cls)
safe_mkdir(os.path.dirname(dst))
os.link(os.path.join(src_output_dir, cls), dst)
dst_deps.save(dst_depfile)
analysis_splits.append(
(sources, dst_output_dir, dst_analysis_cache))
# Use zinc to split the analysis files.
if os.path.exists(src_analysis_cache):
analysis_args = []
analysis_args.extend(self._zinc_jar_args)
analysis_args.extend([
'-log-level',
self.context.options.log_level or 'info',
'-analysis',
])
split_args = analysis_args + [
'-cache',
src_analysis_cache,
'-split',
','.join([
'{%s}:%s' % (':'.join(x[0]), x[2]) for x in analysis_splits
]),
]
if self.runjava(self._main,
classpath=self._zinc_classpath,
args=split_args,
jvmargs=self._jvm_args):
raise TaskError, 'zinc failed to split analysis files %s from %s' %\
(':'.join([x[2] for x in analysis_splits]), src_analysis_cache)
# Now rebase the newly created analysis files.
for split in analysis_splits:
dst_analysis_cache = split[2]
if os.path.exists(dst_analysis_cache):
rebase_args = analysis_args + [
'-cache',
dst_analysis_cache,
'-rebase',
'%s:%s' % (src_output_dir, split[1]),
]
if self.runjava(self._main,
classpath=self._zinc_classpath,
args=rebase_args,
jvmargs=self._jvm_args):
raise TaskError, 'In split_artifact: zinc failed to rebase analysis file %s' % dst_analysis_cache
