def make_zinc_job(target, input_product_key, output_products,
dep_keys):
return Job(
key=self._zinc_key_for_target(target,
rsc_compile_context.workflow),
fn=functools.partial(self._default_work_for_vts, ivts,
zinc_compile_context, input_product_key,
counter, compile_contexts,
CompositeProductAdder(*output_products)),
dependencies=list(dep_keys),
size=self._size_estimator(zinc_compile_context.sources),
)
def make_rsc_job(target, dep_targets):
return Job(
self._rsc_key_for_target(target),
functools.partial(
work_for_vts_rsc,
ivts,
rsc_compile_context),
# The rsc jobs depend on other rsc jobs, and on zinc jobs for targets that are not
# processed by rsc.
list(all_zinc_rsc_invalid_dep_keys(dep_targets)),
self._size_estimator(rsc_compile_context.sources),
)
def create_compile_jobs(self, compile_target, all_compile_contexts, invalid_dependencies, ivts,
counter, classpath_product):
"""Return a list of jobs, and a count of those jobs that represent meaningful ("countable") work."""
context_for_target = all_compile_contexts[compile_target]
compile_context = self.select_runtime_context(context_for_target)
compile_deps = [self.exec_graph_key_for_target(target) for target in invalid_dependencies]
# The cache checking job doesn't technically have any dependencies, but we want to delay it
# until immediately before we would otherwise try compiling, so we indicate that it depends on
# all compile dependencies.
double_check_cache_job = Job(
key=self.exec_graph_double_check_cache_key_for_target(compile_target),
fn=functools.partial(self._default_double_check_cache_for_vts, ivts),
dependencies=compile_deps,
options_scope=self.options_scope,
target=compile_target)
# The compile job depends on the cache check job. This decomposition is necessary in order to
# support more complex situations where compilation runs multiple jobs in parallel, and wants to
# double check the cache before starting any of them.
compile_job = Job(
key=self.exec_graph_key_for_target(compile_target),
fn=functools.partial(
self._default_work_for_vts,
ivts,
compile_context,
'runtime_classpath',
counter,
all_compile_contexts,
classpath_product),
dependencies=[double_check_cache_job.key] + compile_deps,
size=self._size_estimator(compile_context.sources),
# If compilation and analysis work succeeds, validate the vts.
# Otherwise, fail it.
on_success=ivts.update,
on_failure=ivts.force_invalidate,
options_scope=self.options_scope,
target=compile_target)
return ([double_check_cache_job, compile_job], 1)
def job(self,
name,
fn,
dependencies,
size=0,
on_success=None,
on_failure=None):
def recording_fn():
self.jobs_run.append(name)
fn()
return Job(name, recording_fn, dependencies, size, on_success,
on_failure)
def make_rsc_job(target, dep_targets):
return Job(
key=self._rsc_key_for_target(target),
fn=functools.partial(
# NB: This will output to the 'rsc_mixed_compile_classpath' product via
# self.register_extra_products_from_contexts()!
work_for_vts_rsc,
ivts,
rsc_compile_context,
),
# The rsc jobs depend on other rsc jobs, and on zinc jobs for targets that are not
# processed by rsc.
dependencies=list(all_zinc_rsc_invalid_dep_keys(dep_targets)),
size=self._size_estimator(rsc_compile_context.sources),
)
def make_zinc_job(target, input_product_key, output_products,
dep_keys):
return Job(
key=self._zinc_key_for_target(target,
rsc_compile_context.workflow),
fn=functools.partial(self._default_work_for_vts, ivts,
zinc_compile_context, input_product_key,
counter, compile_contexts,
CompositeProductAdder(*output_products)),
dependencies=list(dep_keys),
size=self._size_estimator(zinc_compile_context.sources),
# If compilation and analysis work succeeds, validate the vts.
# Otherwise, fail it.
on_success=ivts.update,
on_failure=ivts.force_invalidate)
def make_outline_job(target, dep_targets):
if workflow == self.JvmCompileWorkflowType.outline_and_zinc:
target_key = self._outline_key_for_target(target)
else:
target_key = self._rsc_key_for_target(target)
return Job(
key=target_key,
fn=functools.partial(
# NB: This will output to the 'rsc_mixed_compile_classpath' product via
# self.register_extra_products_from_contexts()!
work_for_vts_rsc,
ivts,
rsc_compile_context,
),
# The rsc jobs depend on other rsc jobs, and on zinc jobs for targets that are not
# processed by rsc.
dependencies=[cache_doublecheck_key] +
list(all_zinc_rsc_invalid_dep_keys(dep_targets)),
size=self._size_estimator(rsc_compile_context.sources),
options_scope=self.options_scope,
target=target)
def create_compile_jobs(self, compile_target, all_compile_contexts,
invalid_dependencies, ivts, counter,
classpath_product):
context_for_target = all_compile_contexts[compile_target]
compile_context = self.select_runtime_context(context_for_target)
job = Job(
self.exec_graph_key_for_target(compile_target),
functools.partial(self._default_work_for_vts, ivts,
compile_context, 'runtime_classpath', counter,
all_compile_contexts, classpath_product),
[
self.exec_graph_key_for_target(target)
for target in invalid_dependencies
],
self._size_estimator(compile_context.sources),
# If compilation and analysis work succeeds, validate the vts.
# Otherwise, fail it.
on_success=ivts.update,
on_failure=ivts.force_invalidate)
return [job]
def _create_compile_jobs(self, classpath_products, compile_contexts,
extra_compile_time_classpath, invalid_targets,
invalid_vts):
class Counter(object):
def __init__(self, size, initial=0):
self.size = size
self.count = initial
def __call__(self):
self.count += 1
return self.count
def format_length(self):
return len(str(self.size))
counter = Counter(len(invalid_vts))
def check_cache(vts):
"""Manually checks the artifact cache (usually immediately before compilation.)
Returns true if the cache was hit successfully, indicating that no compilation is necessary.
"""
if not self.artifact_cache_reads_enabled():
return False
cached_vts, _, _ = self.check_artifact_cache([vts])
if not cached_vts:
self.context.log.debug(
'Missed cache during double check for {}'.format(
vts.target.address.spec))
return False
assert cached_vts == [
vts
], ('Cache returned unexpected target: {} vs {}'.format(
cached_vts, [vts]))
self.context.log.info(
'Hit cache during double check for {}'.format(
vts.target.address.spec))
counter()
return True
def should_compile_incrementally(vts, ctx):
"""Check to see if the compile should try to re-use the existing analysis.
Returns true if we should try to compile the target incrementally.
"""
if not vts.is_incremental:
return False
if not self._clear_invalid_analysis:
return True
return os.path.exists(ctx.analysis_file)
def work_for_vts(vts, ctx):
progress_message = ctx.target.address.spec
# Capture a compilation log if requested.
log_file = ctx.log_file if self._capture_log else None
# Double check the cache before beginning compilation
hit_cache = check_cache(vts)
if not hit_cache:
# Compute the compile classpath for this target.
cp_entries = [ctx.classes_dir]
cp_entries.extend(
ClasspathUtil.compute_classpath(
ctx.dependencies(self._dep_context),
classpath_products, extra_compile_time_classpath,
self._confs))
upstream_analysis = dict(
self._upstream_analysis(compile_contexts, cp_entries))
if not should_compile_incrementally(vts, ctx):
# Purge existing analysis file in non-incremental mode.
safe_delete(ctx.analysis_file)
# Work around https://github.com/pantsbuild/pants/issues/3670
safe_rmtree(ctx.classes_dir)
tgt, = vts.targets
fatal_warnings = self._compute_language_property(
tgt, lambda x: x.fatal_warnings)
self._compile_vts(vts, ctx.sources, ctx.analysis_file,
upstream_analysis, cp_entries,
ctx.classes_dir, log_file, progress_message,
tgt.platform, fatal_warnings, counter)
self._analysis_tools.relativize(ctx.analysis_file,
ctx.portable_analysis_file)
# Write any additional resources for this target to the target workdir.
self.write_extra_resources(ctx)
# Jar the compiled output.
self._create_context_jar(ctx)
# Update the products with the latest classes.
self._register_vts([ctx])
# Once products are registered, check for unused dependencies (if enabled).
if not hit_cache and self._unused_deps_check_enabled:
self._check_unused_deps(ctx)
jobs = []
invalid_target_set = set(invalid_targets)
for ivts in invalid_vts:
# Invalidated targets are a subset of relevant targets: get the context for this one.
compile_target = ivts.target
compile_context = compile_contexts[compile_target]
invalid_dependencies = self._collect_invalid_compile_dependencies(
compile_target, compile_contexts, invalid_target_set)
jobs.append(
Job(
self.exec_graph_key_for_target(compile_target),
functools.partial(work_for_vts, ivts, compile_context),
[
self.exec_graph_key_for_target(target)
for target in invalid_dependencies
],
self._size_estimator(compile_context.sources),
# If compilation and analysis work succeeds, validate the vts.
# Otherwise, fail it.
on_success=ivts.update,
on_failure=ivts.force_invalidate))
return jobs
def _create_compile_jobs(self, compile_classpaths, compile_contexts,
extra_compile_time_classpath, invalid_targets,
invalid_vts_partitioned, compile_vts,
register_vts, update_artifact_cache_vts_work):
def create_work_for_vts(vts, compile_context, target_closure):
def work():
progress_message = compile_context.target.address.spec
cp_entries = self._compute_classpath_entries(
compile_classpaths, target_closure, compile_context,
extra_compile_time_classpath)
upstream_analysis = dict(
self._upstream_analysis(compile_contexts, cp_entries))
# Capture a compilation log if requested.
log_file = self._capture_log_file(compile_context.target)
# Mutate analysis within a temporary directory, and move it to the final location
# on success.
tmpdir = os.path.join(self.analysis_tmpdir,
compile_context.target.id)
safe_mkdir(tmpdir)
tmp_analysis_file = JvmCompileStrategy._analysis_for_target(
tmpdir, compile_context.target)
if os.path.exists(compile_context.analysis_file):
shutil.copy(compile_context.analysis_file,
tmp_analysis_file)
target, = vts.targets
compile_vts(vts, compile_context.sources, tmp_analysis_file,
upstream_analysis, cp_entries,
compile_context.classes_dir, log_file,
progress_message, target.platform)
atomic_copy(tmp_analysis_file, compile_context.analysis_file)
# Jar the compiled output.
self._create_context_jar(compile_context)
# Update the products with the latest classes.
register_vts([compile_context])
# Kick off the background artifact cache write.
if update_artifact_cache_vts_work:
self._write_to_artifact_cache(
vts, compile_context, update_artifact_cache_vts_work)
return work
jobs = []
invalid_target_set = set(invalid_targets)
for vts in invalid_vts_partitioned:
assert len(vts.targets) == 1, (
"Requested one target per partition, got {}".format(vts))
# Invalidated targets are a subset of relevant targets: get the context for this one.
compile_target = vts.targets[0]
compile_context = compile_contexts[compile_target]
compile_target_closure = compile_target.closure()
# dependencies of the current target which are invalid for this chunk
invalid_dependencies = (compile_target_closure
& invalid_target_set) - [compile_target]
jobs.append(
Job(
self.exec_graph_key_for_target(compile_target),
create_work_for_vts(vts, compile_context,
compile_target_closure),
[
self.exec_graph_key_for_target(target)
for target in invalid_dependencies
],
self._size_estimator(compile_context.sources),
# If compilation and analysis work succeeds, validate the vts.
# Otherwise, fail it.
on_success=vts.update,
on_failure=vts.force_invalidate))
return jobs
def create_compile_jobs(self, compile_target, all_compile_contexts, invalid_dependencies, ivts,
counter):
def work_for_vts(vts, ctx):
progress_message = ctx.target.address.spec
# Double check the cache before beginning compilation
hit_cache = self.check_cache(vts, counter)
if not hit_cache:
# Compute the compile classpath for this target.
cp_entries = self._cp_entries_for_ctx(ctx, 'runtime_classpath')
upstream_analysis = dict(self._upstream_analysis(all_compile_contexts, cp_entries))
is_incremental = self.should_compile_incrementally(vts, ctx)
if not is_incremental:
# Purge existing analysis file in non-incremental mode.
safe_delete(ctx.analysis_file)
# Work around https://github.com/pantsbuild/pants/issues/3670
safe_rmtree(ctx.classes_dir)
dep_context = DependencyContext.global_instance()
tgt, = vts.targets
compiler_option_sets = dep_context.defaulted_property(tgt, lambda x: x.compiler_option_sets)
zinc_file_manager = dep_context.defaulted_property(tgt, lambda x: x.zinc_file_manager)
with Timer() as timer:
self._compile_vts(vts,
ctx,
upstream_analysis,
cp_entries,
progress_message,
tgt.platform,
compiler_option_sets,
zinc_file_manager,
counter)
self._record_target_stats(tgt,
len(cp_entries),
len(ctx.sources),
timer.elapsed,
is_incremental,
'compile')
# Write any additional resources for this target to the target workdir.
self.write_extra_resources(ctx)
# Jar the compiled output.
self._create_context_jar(ctx)
# Update the products with the latest classes.
self.register_extra_products_from_contexts([ctx.target], all_compile_contexts)
context_for_target = all_compile_contexts[compile_target]
compile_context = self.select_runtime_context(context_for_target)
job = Job(self.exec_graph_key_for_target(compile_target),
functools.partial(work_for_vts, ivts, compile_context),
[self.exec_graph_key_for_target(target) for target in invalid_dependencies],
self._size_estimator(compile_context.sources),
# If compilation and analysis work succeeds, validate the vts.
# Otherwise, fail it.
on_success=ivts.update,
on_failure=ivts.force_invalidate)
return [job]
def _create_compile_jobs(self, classpath_products, compile_contexts,
extra_compile_time_classpath, invalid_targets,
invalid_vts):
class Counter(object):
def __init__(self, size, initial=0):
self.size = size
self.count = initial
def __call__(self):
self.count += 1
return self.count
def format_length(self):
return len(str(self.size))
counter = Counter(len(invalid_vts))
def check_cache(vts):
"""Manually checks the artifact cache (usually immediately before compilation.)
Returns true if the cache was hit successfully, indicating that no compilation is necessary.
"""
if not self.artifact_cache_reads_enabled():
return False
cached_vts, _, _ = self.check_artifact_cache([vts])
if not cached_vts:
self.context.log.debug(
'Missed cache during double check for {}'.format(
vts.target.address.spec))
return False
assert cached_vts == [
vts
], ('Cache returned unexpected target: {} vs {}'.format(
cached_vts, [vts]))
self.context.log.info(
'Hit cache during double check for {}'.format(
vts.target.address.spec))
counter()
return True
def should_compile_incrementally(vts):
"""Check to see if the compile should try to re-use the existing analysis.
Returns true if we should try to compile the target incrementally.
"""
if not vts.is_incremental:
return False
if not self._clear_invalid_analysis:
return True
return os.path.exists(compile_context.analysis_file)
def work_for_vts(vts, ctx):
progress_message = ctx.target.address.spec
# Capture a compilation log if requested.
log_file = ctx.log_file if self._capture_log else None
# Double check the cache before beginning compilation
hit_cache = check_cache(vts)
if not hit_cache:
# Compute the compile classpath for this target.
cp_entries = self._compute_classpath_entries(
classpath_products, ctx, extra_compile_time_classpath)
# TODO: always provide transitive analysis, but not always all classpath entries?
upstream_analysis = dict(
self._upstream_analysis(compile_contexts, cp_entries))
# Write analysis to a temporary file, and move it to the final location on success.
tmp_analysis_file = "{}.tmp".format(ctx.analysis_file)
if should_compile_incrementally(vts):
# If this is an incremental compile, rebase the analysis to our new classes directory.
self._analysis_tools.rebase_from_path(
ctx.analysis_file, tmp_analysis_file,
vts.previous_results_dir, vts.results_dir)
else:
# Otherwise, simply ensure that it is empty.
safe_delete(tmp_analysis_file)
tgt, = vts.targets
fatal_warnings = self._compute_language_property(
tgt, lambda x: x.fatal_warnings)
self._compile_vts(vts, ctx.sources, tmp_analysis_file,
upstream_analysis, cp_entries,
ctx.classes_dir, log_file, progress_message,
tgt.platform, fatal_warnings, counter)
os.rename(tmp_analysis_file, ctx.analysis_file)
self._analysis_tools.relativize(ctx.analysis_file,
ctx.portable_analysis_file)
# Write any additional resources for this target to the target workdir.
self.write_extra_resources(ctx)
# Jar the compiled output.
self._create_context_jar(ctx)
# Update the products with the latest classes.
self._register_vts([ctx])
jobs = []
invalid_target_set = set(invalid_targets)
for ivts in invalid_vts:
# Invalidated targets are a subset of relevant targets: get the context for this one.
compile_target = ivts.targets[0]
compile_context = compile_contexts[compile_target]
compile_target_closure = compile_target.closure()
# dependencies of the current target which are invalid for this chunk
invalid_dependencies = (compile_target_closure
& invalid_target_set) - [compile_target]
jobs.append(
Job(
self.exec_graph_key_for_target(compile_target),
functools.partial(work_for_vts, ivts, compile_context),
[
self.exec_graph_key_for_target(target)
for target in invalid_dependencies
],
self._size_estimator(compile_context.sources),
# If compilation and analysis work succeeds, validate the vts.
# Otherwise, fail it.
on_success=ivts.update,
on_failure=ivts.force_invalidate))
return jobs
def _create_compile_jobs(self, classpath_products, compile_contexts,
extra_compile_time_classpath, invalid_targets,
invalid_vts_partitioned):
def check_cache(vts):
"""Manually checks the artifact cache (usually immediately before compilation.)
Returns true if the cache was hit successfully, indicating that no compilation is necessary.
"""
if not self.artifact_cache_reads_enabled():
return False
cached_vts, uncached_vts = self.check_artifact_cache([vts])
if not cached_vts:
self.context.log.debug(
'Missed cache during double check for {}'.format(
vts.target.address.spec))
return False
assert cached_vts == [
vts
], ('Cache returned unexpected target: {} vs {}'.format(
cached_vts, [vts]))
self.context.log.info(
'Hit cache during double check for {}'.format(
vts.target.address.spec))
return True
def work_for_vts(vts, compile_context):
progress_message = compile_context.target.address.spec
cp_entries = self._compute_classpath_entries(
classpath_products, compile_context,
extra_compile_time_classpath)
upstream_analysis = dict(
self._upstream_analysis(compile_contexts, cp_entries))
# Capture a compilation log if requested.
log_file = compile_context.log_file if self._capture_log else None
# Double check the cache before beginning compilation
hit_cache = check_cache(vts)
if not hit_cache:
# Write analysis to a temporary file, and move it to the final location on success.
tmp_analysis_file = "{}.tmp".format(
compile_context.analysis_file)
if vts.is_incremental:
# If this is an incremental compile, rebase the analysis to our new classes directory.
self._analysis_tools.rebase_from_path(
compile_context.analysis_file, tmp_analysis_file,
vts.previous_results_dir, vts.results_dir)
else:
# Otherwise, simply ensure that it is empty.
safe_delete(tmp_analysis_file)
target, = vts.targets
self._compile_vts(vts, compile_context.sources,
tmp_analysis_file, upstream_analysis,
cp_entries, compile_context.classes_dir,
log_file, progress_message, target.platform)
os.rename(tmp_analysis_file, compile_context.analysis_file)
self._analysis_tools.relativize(
compile_context.analysis_file,
compile_context.portable_analysis_file)
# Write any additional resources for this target to the target workdir.
self.write_extra_resources(compile_context)
# Jar the compiled output.
self._create_context_jar(compile_context)
# Update the products with the latest classes.
self._register_vts([compile_context])
jobs = []
invalid_target_set = set(invalid_targets)
for vts in invalid_vts_partitioned:
assert len(vts.targets) == 1, (
"Requested one target per partition, got {}".format(vts))
# Invalidated targets are a subset of relevant targets: get the context for this one.
compile_target = vts.targets[0]
compile_context = compile_contexts[compile_target]
compile_target_closure = compile_target.closure()
# dependencies of the current target which are invalid for this chunk
invalid_dependencies = (compile_target_closure
& invalid_target_set) - [compile_target]
jobs.append(
Job(
self.exec_graph_key_for_target(compile_target),
functools.partial(work_for_vts, vts, compile_context),
[
self.exec_graph_key_for_target(target)
for target in invalid_dependencies
],
self._size_estimator(compile_context.sources),
# If compilation and analysis work succeeds, validate the vts.
# Otherwise, fail it.
on_success=vts.update,
on_failure=vts.force_invalidate))
return jobs
def _create_compile_jobs(self, classpath_products, compile_contexts,
extra_compile_time_classpath, invalid_targets,
invalid_vts_partitioned, check_vts, compile_vts,
register_vts, update_artifact_cache_vts_work):
def check_cache(vts):
"""Manually checks the artifact cache (usually immediately before compilation.)
Returns true if the cache was hit successfully, indicating that no compilation is necessary.
"""
if not check_vts:
return False
cached_vts, uncached_vts = check_vts([vts])
if not cached_vts:
self.context.log.debug(
'Missed cache during double check for {}'.format(
vts.target.address.spec))
return False
assert cached_vts == [
vts
], ('Cache returned unexpected target: {} vs {}'.format(
cached_vts, [vts]))
self.context.log.info(
'Hit cache during double check for {}'.format(
vts.target.address.spec))
return True
def work_for_vts(vts, compile_context, target_closure):
progress_message = compile_context.target.address.spec
cp_entries = self._compute_classpath_entries(
classpath_products, target_closure, compile_context,
extra_compile_time_classpath)
upstream_analysis = dict(
self._upstream_analysis(compile_contexts, cp_entries))
# Capture a compilation log if requested.
log_file = self._capture_log_file(compile_context.target)
# Double check the cache before beginning compilation
hit_cache = check_cache(vts)
incremental = False
if not hit_cache:
# Mutate analysis within a temporary directory, and move it to the final location
# on success.
tmpdir = os.path.join(self.analysis_tmpdir,
compile_context.target.id)
safe_mkdir(tmpdir)
tmp_analysis_file = self._analysis_for_target(
tmpdir, compile_context.target)
# If the analysis exists for this context, it is an incremental compile.
if os.path.exists(compile_context.analysis_file):
incremental = True
shutil.copy(compile_context.analysis_file,
tmp_analysis_file)
target, = vts.targets
compile_vts(vts, compile_context.sources, tmp_analysis_file,
upstream_analysis, cp_entries,
compile_context.classes_dir, log_file,
progress_message, target.platform)
atomic_copy(tmp_analysis_file, compile_context.analysis_file)
# Jar the compiled output.
self._create_context_jar(compile_context)
# Update the products with the latest classes.
register_vts([compile_context])
# We write to the cache only if we didn't hit during the double check, and optionally
# only for clean builds.
is_cacheable = not hit_cache and (
self.get_options().incremental_caching or not incremental)
self.context.log.debug(
'Completed compile for {}. '
'Hit cache: {}, was incremental: {}, is cacheable: {}, cache writes enabled: {}.'
.format(compile_context.target.address.spec, hit_cache,
incremental, is_cacheable,
update_artifact_cache_vts_work is not None))
if is_cacheable and update_artifact_cache_vts_work:
# Kick off the background artifact cache write.
self._write_to_artifact_cache(vts, compile_context,
update_artifact_cache_vts_work)
jobs = []
invalid_target_set = set(invalid_targets)
for vts in invalid_vts_partitioned:
assert len(vts.targets) == 1, (
"Requested one target per partition, got {}".format(vts))
# Invalidated targets are a subset of relevant targets: get the context for this one.
compile_target = vts.targets[0]
compile_context = compile_contexts[compile_target]
compile_target_closure = compile_target.closure()
# dependencies of the current target which are invalid for this chunk
invalid_dependencies = (compile_target_closure
& invalid_target_set) - [compile_target]
jobs.append(
Job(
self.exec_graph_key_for_target(compile_target),
functools.partial(work_for_vts, vts, compile_context,
compile_target_closure),
[
self.exec_graph_key_for_target(target)
for target in invalid_dependencies
],
self._size_estimator(compile_context.sources),
# If compilation and analysis work succeeds, validate the vts.
# Otherwise, fail it.
on_success=vts.update,
on_failure=vts.force_invalidate))
return jobs
def create_compile_jobs(self,
compile_target,
compile_contexts,
invalid_dependencies,
ivts,
counter,
runtime_classpath_product):
def work_for_vts_rsc(vts, ctx):
# Double check the cache before beginning compilation
hit_cache = self.check_cache(vts, counter)
target = ctx.target
if not hit_cache:
cp_entries = []
# Include the current machine's jdk lib jars. This'll blow up remotely.
# We need a solution for that.
# Probably something to do with https://github.com/pantsbuild/pants/pull/6346
distribution = JvmPlatform.preferred_jvm_distribution([ctx.target.platform], strict=True)
jvm_lib_jars_abs = distribution.find_libs(['rt.jar', 'dt.jar', 'jce.jar', 'tools.jar'])
cp_entries.extend(jvm_lib_jars_abs)
classpath_abs = self._zinc.compile_classpath(
'rsc_classpath',
ctx.target,
extra_cp_entries=self._extra_compile_time_classpath)
jar_deps = [t for t in DependencyContext.global_instance().dependencies_respecting_strict_deps(target)
if isinstance(t, JarLibrary)]
metacp_jar_classpath_abs = [y[1] for y in self._metacp_jars_classpath_product.get_for_targets(
jar_deps
)]
jar_jar_paths = {y[1] for y in self.context.products.get_data('rsc_classpath').get_for_targets(jar_deps)}
classpath_abs = [c for c in classpath_abs if c not in jar_jar_paths]
classpath_rel = fast_relpath_collection(classpath_abs)
metacp_jar_classpath_rel = fast_relpath_collection(metacp_jar_classpath_abs)
cp_entries.extend(classpath_rel)
ctx.ensure_output_dirs_exist()
counter_val = str(counter()).rjust(counter.format_length(), b' ')
counter_str = '[{}/{}] '.format(counter_val, counter.size)
self.context.log.info(
counter_str,
'Rsc-ing ',
items_to_report_element(ctx.sources, '{} source'.format(self.name())),
' in ',
items_to_report_element([t.address.reference() for t in vts.targets], 'target'),
' (',
ctx.target.address.spec,
').')
tgt, = vts.targets
with Timer() as timer:
# Step 1: Convert classpath to SemanticDB
# ---------------------------------------
scalac_classpath_path_entries_abs = self.tool_classpath('workaround-metacp-dependency-classpath')
scalac_classpath_path_entries = fast_relpath_collection(scalac_classpath_path_entries_abs)
rsc_index_dir = fast_relpath(ctx.rsc_index_dir, get_buildroot())
args = [
'--verbose',
# NB: Without this setting, rsc will be missing some symbols
# from the scala library.
'--include-scala-library-synthetics', # TODO generate these once and cache them
# NB: We need to add these extra dependencies in order to be able
# to find symbols used by the scalac jars.
'--dependency-classpath', os.pathsep.join(scalac_classpath_path_entries + list(jar_jar_paths)),
# NB: The directory to dump the semanticdb jars generated by metacp.
'--out', rsc_index_dir,
os.pathsep.join(cp_entries),
]
metacp_wu = self._runtool(
'scala.meta.cli.Metacp',
'metacp',
args,
distribution,
tgt=tgt,
input_files=(scalac_classpath_path_entries + classpath_rel),
output_dir=rsc_index_dir)
metacp_stdout = stdout_contents(metacp_wu)
metacp_result = json.loads(metacp_stdout)
metai_classpath = self._collect_metai_classpath(
metacp_result, classpath_rel, jvm_lib_jars_abs)
# Step 1.5: metai Index the semanticdbs
# -------------------------------------
self._run_metai_tool(distribution, metai_classpath, rsc_index_dir, tgt)
# Step 2: Outline Scala sources into SemanticDB
# ---------------------------------------------
rsc_outline_dir = fast_relpath(ctx.rsc_outline_dir, get_buildroot())
rsc_out = os.path.join(rsc_outline_dir, 'META-INF/semanticdb/out.semanticdb')
safe_mkdir(os.path.join(rsc_outline_dir, 'META-INF/semanticdb'))
target_sources = ctx.sources
args = [
'-cp', os.pathsep.join(metai_classpath + metacp_jar_classpath_rel),
'-out', rsc_out,
] + target_sources
self._runtool(
'rsc.cli.Main',
'rsc',
args,
distribution,
tgt=tgt,
# TODO pass the input files from the target snapshot instead of the below
# input_snapshot = ctx.target.sources_snapshot(scheduler=self.context._scheduler)
input_files=target_sources + metai_classpath + metacp_jar_classpath_rel,
output_dir=rsc_outline_dir)
rsc_classpath = [rsc_outline_dir]
# Step 2.5: Postprocess the rsc outputs
# TODO: This is only necessary as a workaround for https://github.com/twitter/rsc/issues/199.
# Ideally, Rsc would do this on its own.
self._run_metai_tool(distribution,
rsc_classpath,
rsc_outline_dir,
tgt,
extra_input_files=(rsc_out,))
# Step 3: Convert SemanticDB into an mjar
# ---------------------------------------
rsc_mjar_file = fast_relpath(ctx.rsc_mjar_file, get_buildroot())
args = [
'-out', rsc_mjar_file,
os.pathsep.join(rsc_classpath),
]
self._runtool(
'scala.meta.cli.Mjar',
'mjar',
args,
distribution,
tgt=tgt,
input_files=(
rsc_out,
),
output_dir=os.path.dirname(rsc_mjar_file)
)
self.context.products.get_data('rsc_classpath').add_for_target(
ctx.target,
[(conf, ctx.rsc_mjar_file) for conf in self._confs],
)
self._record_target_stats(tgt,
len(cp_entries),
len(target_sources),
timer.elapsed,
False,
'rsc'
)
# Write any additional resources for this target to the target workdir.
self.write_extra_resources(ctx)
# Update the products with the latest classes.
self.register_extra_products_from_contexts([ctx.target], compile_contexts)
def work_for_vts_rsc_jar_library(vts, ctx):
cp_entries = []
# Include the current machine's jdk lib jars. This'll blow up remotely.
# We need a solution for that.
# Probably something to do with https://github.com/pantsbuild/pants/pull/6346
# TODO perhaps determine the platform of the jar and use that here.
# https://github.com/pantsbuild/pants/issues/6547
distribution = JvmPlatform.preferred_jvm_distribution([], strict=True)
jvm_lib_jars_abs = distribution.find_libs(['rt.jar', 'dt.jar', 'jce.jar', 'tools.jar'])
cp_entries.extend(jvm_lib_jars_abs)
# TODO use compile_classpath
classpath_abs = [
path for (conf, path) in
self.context.products.get_data('rsc_classpath').get_for_target(ctx.target)
]
dependency_classpath = self._zinc.compile_classpath(
'compile_classpath',
ctx.target,
extra_cp_entries=self._extra_compile_time_classpath)
classpath_rel = fast_relpath_collection(classpath_abs)
cp_entries.extend(classpath_rel)
counter_val = str(counter()).rjust(counter.format_length(), b' ')
counter_str = '[{}/{}] '.format(counter_val, counter.size)
self.context.log.info(
counter_str,
'Metacp-ing ',
items_to_report_element(cp_entries, 'jar'),
' in ',
items_to_report_element([t.address.reference() for t in vts.targets], 'target'),
' (',
ctx.target.address.spec,
').')
ctx.ensure_output_dirs_exist()
tgt, = vts.targets
with Timer() as timer:
# Step 1: Convert classpath to SemanticDB
# ---------------------------------------
scalac_classpath_path_entries_abs = self.tool_classpath('workaround-metacp-dependency-classpath')
scalac_classpath_path_entries = fast_relpath_collection(scalac_classpath_path_entries_abs)
rsc_index_dir = fast_relpath(ctx.rsc_index_dir, get_buildroot())
args = [
'--verbose',
# NB: Without this setting, rsc will be missing some symbols
# from the scala library.
'--include-scala-library-synthetics', # TODO generate these once and cache them
# NB: We need to add these extra dependencies in order to be able
# to find symbols used by the scalac jars.
'--dependency-classpath', os.pathsep.join(dependency_classpath + scalac_classpath_path_entries),
# NB: The directory to dump the semanticdb jars generated by metacp.
'--out', rsc_index_dir,
os.pathsep.join(cp_entries),
]
metacp_wu = self._runtool(
'scala.meta.cli.Metacp',
'metacp',
args,
distribution,
tgt=tgt,
input_files=(scalac_classpath_path_entries + classpath_rel),
output_dir=rsc_index_dir)
metacp_stdout = stdout_contents(metacp_wu)
metacp_result = json.loads(metacp_stdout)
metai_classpath = self._collect_metai_classpath(
metacp_result, classpath_rel, jvm_lib_jars_abs)
# Step 1.5: metai Index the semanticdbs
# -------------------------------------
self._run_metai_tool(distribution, metai_classpath, rsc_index_dir, tgt)
abs_output = [(conf, os.path.join(get_buildroot(), x))
for conf in self._confs for x in metai_classpath]
self._metacp_jars_classpath_product.add_for_target(
ctx.target,
abs_output,
)
self._record_target_stats(tgt,
len(abs_output),
len([]),
timer.elapsed,
False,
'metacp'
)
rsc_jobs = []
zinc_jobs = []
# Invalidated targets are a subset of relevant targets: get the context for this one.
compile_target = ivts.target
compile_context_pair = compile_contexts[compile_target]
# Create the rsc job.
# Currently, rsc only supports outlining scala.
if self._only_zinc_compileable(compile_target):
pass
elif self._rsc_compilable(compile_target):
rsc_key = self._rsc_key_for_target(compile_target)
rsc_jobs.append(
Job(
rsc_key,
functools.partial(
work_for_vts_rsc,
ivts,
compile_context_pair[0]),
[self._rsc_key_for_target(target) for target in invalid_dependencies],
self._size_estimator(compile_context_pair[0].sources),
)
)
elif self._metacpable(compile_target):
rsc_key = self._rsc_key_for_target(compile_target)
rsc_jobs.append(
Job(
rsc_key,
functools.partial(
work_for_vts_rsc_jar_library,
ivts,
compile_context_pair[0]),
[self._rsc_key_for_target(target) for target in invalid_dependencies],
self._size_estimator(compile_context_pair[0].sources),
on_success=ivts.update,
on_failure=ivts.force_invalidate,
)
)
else:
raise TaskError("Unexpected target for rsc compile {} with type {}"
.format(compile_target, type(compile_target)))
# Create the zinc compile jobs.
# - Scala zinc compile jobs depend on the results of running rsc on the scala target.
# - Java zinc compile jobs depend on the zinc compiles of their dependencies, because we can't
# generate mjars that make javac happy at this point.
invalid_dependencies_without_jar_metacps = [t for t in invalid_dependencies
if not self._metacpable(t)]
if self._rsc_compilable(compile_target):
full_key = self._compile_against_rsc_key_for_target(compile_target)
zinc_jobs.append(
Job(
full_key,
functools.partial(
self._default_work_for_vts,
ivts,
compile_context_pair[1],
'rsc_classpath',
counter,
compile_contexts,
runtime_classpath_product),
[
self._rsc_key_for_target(compile_target)
] + [
self._rsc_key_for_target(target)
for target in invalid_dependencies_without_jar_metacps
],
self._size_estimator(compile_context_pair[1].sources),
# NB: right now, only the last job will write to the cache, because we don't
# do multiple cache entries per target-task tuple.
on_success=ivts.update,
on_failure=ivts.force_invalidate,
)
)
elif self._only_zinc_compileable(compile_target):
# write to both rsc classpath and runtime classpath
class CompositeProductAdder(object):
def __init__(self, runtime_classpath_product, rsc_classpath_product):
self.rsc_classpath_product = rsc_classpath_product
self.runtime_classpath_product = runtime_classpath_product
def add_for_target(self, *args, **kwargs):
self.runtime_classpath_product.add_for_target(*args, **kwargs)
self.rsc_classpath_product.add_for_target(*args, **kwargs)
full_key = self._compile_against_rsc_key_for_target(compile_target)
zinc_jobs.append(
Job(
full_key,
functools.partial(
self._default_work_for_vts,
ivts,
compile_context_pair[1],
'runtime_classpath',
counter,
compile_contexts,
CompositeProductAdder(
runtime_classpath_product,
self.context.products.get_data('rsc_classpath'))),
[
self._compile_against_rsc_key_for_target(target)
for target in invalid_dependencies_without_jar_metacps],
self._size_estimator(compile_context_pair[1].sources),
# NB: right now, only the last job will write to the cache, because we don't
# do multiple cache entries per target-task tuple.
on_success=ivts.update,
on_failure=ivts.force_invalidate,
)
)
return rsc_jobs + zinc_jobs
def create_compile_jobs(
self,
compile_target,
compile_contexts,
invalid_dependencies,
ivts,
counter,
runtime_classpath_product,
):
def work_for_vts_rsc(vts, ctx):
target = ctx.target
(tgt, ) = vts.targets
rsc_cc = compile_contexts[target].rsc_cc
use_youtline = rsc_cc.workflow == self.JvmCompileWorkflowType.outline_and_zinc
outliner = "scalac-outliner" if use_youtline else "rsc"
if use_youtline and Semver.parse(
self._scala_library_version) < Semver.parse("2.12.9"):
raise RuntimeError(
f"To use scalac's built-in outlining, scala version must be at least 2.12.9, but got {self._scala_library_version}"
)
# If we didn't hit the cache in the cache job, run rsc.
if not vts.valid:
counter_val = str(counter()).rjust(counter.format_length(),
" ")
counter_str = f"[{counter_val}/{counter.size}] "
action_str = "Outlining " if use_youtline else "Rsc-ing "
self.context.log.info(
counter_str,
action_str,
items_to_report_element(ctx.sources,
f"{self.name()} source"),
" in ",
items_to_report_element(
[t.address.reference() for t in vts.targets],
"target"),
" (",
ctx.target.address.spec,
").",
)
# This does the following
# - Collect the rsc classpath elements, including zinc compiles of rsc incompatible targets
# and rsc compiles of rsc compatible targets.
# - Run Rsc on the current target with those as dependencies.
dependencies_for_target = list(
DependencyContext.global_instance(
).dependencies_respecting_strict_deps(target))
classpath_paths = []
classpath_digests = []
classpath_product = self.context.products.get_data(
"rsc_mixed_compile_classpath")
classpath_entries = classpath_product.get_classpath_entries_for_targets(
dependencies_for_target)
hermetic = self.execution_strategy == self.ExecutionStrategy.hermetic
for _conf, classpath_entry in classpath_entries:
classpath_paths.append(
fast_relpath(classpath_entry.path, get_buildroot()))
if hermetic and not classpath_entry.directory_digest:
raise AssertionError(
"ClasspathEntry {} didn't have a Digest, so won't be present for hermetic "
"execution of {}".format(classpath_entry,
outliner))
classpath_digests.append(classpath_entry.directory_digest)
ctx.ensure_output_dirs_exist()
with Timer() as timer:
# Outline Scala sources into SemanticDB / scalac compatible header jars.
# ---------------------------------------------
rsc_jar_file_relative_path = fast_relpath(
ctx.rsc_jar_file.path, get_buildroot())
sources_snapshot = ctx.target.sources_snapshot(
scheduler=self.context._scheduler)
distribution = self._get_jvm_distribution()
def hermetic_digest_classpath():
jdk_libs_rel, jdk_libs_digest = self._jdk_libs_paths_and_digest(
distribution)
merged_sources_and_jdk_digest = self.context._scheduler.merge_directories(
(jdk_libs_digest, sources_snapshot.digest) +
tuple(classpath_digests))
classpath_rel_jdk = classpath_paths + jdk_libs_rel
return (merged_sources_and_jdk_digest,
classpath_rel_jdk)
def nonhermetic_digest_classpath():
classpath_abs_jdk = classpath_paths + self._jdk_libs_abs(
distribution)
return ((EMPTY_DIGEST), classpath_abs_jdk)
(input_digest, classpath_entry_paths) = match(
self.execution_strategy,
{
self.ExecutionStrategy.hermetic:
hermetic_digest_classpath,
self.ExecutionStrategy.subprocess:
nonhermetic_digest_classpath,
self.ExecutionStrategy.nailgun:
nonhermetic_digest_classpath,
},
)()
youtline_args = []
if use_youtline:
youtline_args = [
"-Youtline",
"-Ystop-after:pickler",
"-Ypickle-write",
rsc_jar_file_relative_path,
]
target_sources = ctx.sources
# TODO: m.jar digests aren't found, so hermetic will fail.
if use_youtline and not hermetic and self.get_options(
).zinc_outline:
self._zinc_outline(ctx, classpath_paths,
target_sources, youtline_args)
else:
args = ([
"-cp",
os.pathsep.join(classpath_entry_paths),
"-d",
rsc_jar_file_relative_path,
] + self.get_options().extra_rsc_args + youtline_args +
target_sources)
self.write_argsfile(ctx, args)
self._runtool(distribution, input_digest, ctx,
use_youtline)
self._record_target_stats(
tgt,
len(classpath_entry_paths),
len(target_sources),
timer.elapsed,
False,
outliner,
)
# Update the products with the latest classes.
self.register_extra_products_from_contexts([ctx.target],
compile_contexts)
### Create Jobs for ExecutionGraph
cache_doublecheck_jobs = []
rsc_jobs = []
zinc_jobs = []
# Invalidated targets are a subset of relevant targets: get the context for this one.
compile_target = ivts.target
merged_compile_context = compile_contexts[compile_target]
rsc_compile_context = merged_compile_context.rsc_cc
zinc_compile_context = merged_compile_context.zinc_cc
workflow = rsc_compile_context.workflow
cache_doublecheck_key = self.exec_graph_double_check_cache_key_for_target(
compile_target)
def all_zinc_rsc_invalid_dep_keys(invalid_deps):
"""Get the rsc key for an rsc-and-zinc target, or the zinc key for a zinc-only
target."""
for tgt in invalid_deps:
# None can occur for e.g. JarLibrary deps, which we don't need to compile as they are
# populated in the resolve goal.
tgt_rsc_cc = compile_contexts[tgt].rsc_cc
if tgt_rsc_cc.workflow is not None:
# Rely on the results of zinc compiles for zinc-compatible targets
yield self._key_for_target_as_dep(tgt, tgt_rsc_cc.workflow)
def make_cache_doublecheck_job(dep_keys):
# As in JvmCompile.create_compile_jobs, we create a cache-double-check job that all "real" work
# depends on. It depends on completion of the same dependencies as the rsc job in order to run
# as late as possible, while still running before rsc or zinc.
return Job(
key=cache_doublecheck_key,
fn=functools.partial(self._double_check_cache_for_vts, ivts,
zinc_compile_context),
dependencies=list(dep_keys),
options_scope=self.options_scope,
)
def make_outline_job(target, dep_targets):
if workflow == self.JvmCompileWorkflowType.outline_and_zinc:
target_key = self._outline_key_for_target(target)
else:
target_key = self._rsc_key_for_target(target)
return Job(
key=target_key,
fn=functools.partial(
# NB: This will output to the 'rsc_mixed_compile_classpath' product via
# self.register_extra_products_from_contexts()!
work_for_vts_rsc,
ivts,
rsc_compile_context,
),
# The rsc jobs depend on other rsc jobs, and on zinc jobs for targets that are not
# processed by rsc.
dependencies=[cache_doublecheck_key] +
list(all_zinc_rsc_invalid_dep_keys(dep_targets)),
size=self._size_estimator(rsc_compile_context.sources),
options_scope=self.options_scope,
target=target,
)
def only_zinc_invalid_dep_keys(invalid_deps):
for tgt in invalid_deps:
rsc_cc_tgt = compile_contexts[tgt].rsc_cc
if rsc_cc_tgt.workflow is not None:
yield self._zinc_key_for_target(tgt, rsc_cc_tgt.workflow)
def make_zinc_job(target, input_product_key, output_products,
dep_keys):
return Job(
key=self._zinc_key_for_target(target,
rsc_compile_context.workflow),
fn=functools.partial(
self._default_work_for_vts,
ivts,
zinc_compile_context,
input_product_key,
counter,
compile_contexts,
CompositeProductAdder(*output_products),
),
dependencies=[cache_doublecheck_key] + list(dep_keys),
size=self._size_estimator(zinc_compile_context.sources),
options_scope=self.options_scope,
target=target,
)
# Replica of JvmCompile's _record_target_stats logic
def record(k, v):
self.context.run_tracker.report_target_info(
self.options_scope, compile_target, ["compile", k], v)
record("workflow", workflow.value)
record("execution_strategy", self.execution_strategy.value)
# Create the cache doublecheck job.
match(
workflow,
{
self.JvmCompileWorkflowType.zinc_only:
lambda: cache_doublecheck_jobs.append(
make_cache_doublecheck_job(
list(
all_zinc_rsc_invalid_dep_keys(invalid_dependencies)
))),
self.JvmCompileWorkflowType.zinc_java:
lambda: cache_doublecheck_jobs.append(
make_cache_doublecheck_job(
list(only_zinc_invalid_dep_keys(invalid_dependencies)))
),
self.JvmCompileWorkflowType.rsc_and_zinc:
lambda: cache_doublecheck_jobs.append(
make_cache_doublecheck_job(
list(
all_zinc_rsc_invalid_dep_keys(invalid_dependencies)
))),
self.JvmCompileWorkflowType.outline_and_zinc:
lambda: cache_doublecheck_jobs.append(
make_cache_doublecheck_job(
list(
all_zinc_rsc_invalid_dep_keys(invalid_dependencies)
))),
},
)()
# Create the rsc job.
# Currently, rsc only supports outlining scala.
match(
workflow,
{
self.JvmCompileWorkflowType.zinc_only:
lambda: None,
self.JvmCompileWorkflowType.zinc_java:
lambda: None,
self.JvmCompileWorkflowType.rsc_and_zinc:
lambda: rsc_jobs.append(
make_outline_job(compile_target, invalid_dependencies)),
self.JvmCompileWorkflowType.outline_and_zinc:
lambda: rsc_jobs.append(
make_outline_job(compile_target, invalid_dependencies)),
},
)()
# Create the zinc compile jobs.
# - Scala zinc compile jobs depend on the results of running rsc on the scala target.
# - Java zinc compile jobs depend on the zinc compiles of their dependencies, because we can't
# generate jars that make javac happy at this point.
match(
workflow,
{
# NB: zinc-only zinc jobs run zinc and depend on rsc and/or zinc compile outputs.
self.JvmCompileWorkflowType.zinc_only:
lambda: zinc_jobs.append(
make_zinc_job(
compile_target,
input_product_key="rsc_mixed_compile_classpath",
output_products=[
runtime_classpath_product,
self.context.products.get_data(
"rsc_mixed_compile_classpath"),
],
dep_keys=list(
all_zinc_rsc_invalid_dep_keys(invalid_dependencies)
),
)),
# NB: javac can't read rsc output yet, so we need it to depend strictly on zinc
# compilations of dependencies.
self.JvmCompileWorkflowType.zinc_java:
lambda: zinc_jobs.append(
make_zinc_job(
compile_target,
input_product_key="runtime_classpath",
output_products=[
runtime_classpath_product,
self.context.products.get_data(
"rsc_mixed_compile_classpath"),
],
dep_keys=list(
only_zinc_invalid_dep_keys(invalid_dependencies)),
)),
self.JvmCompileWorkflowType.rsc_and_zinc:
lambda: zinc_jobs.append(
# NB: rsc-and-zinc jobs run zinc and depend on both rsc and zinc compile outputs.
make_zinc_job(
compile_target,
input_product_key="rsc_mixed_compile_classpath",
# NB: We want to ensure the 'runtime_classpath' product *only* contains the outputs of
# zinc compiles, and that the 'rsc_mixed_compile_classpath' entries for rsc-compatible targets
# *only* contain the output of an rsc compile for that target.
output_products=[runtime_classpath_product],
dep_keys=list(
all_zinc_rsc_invalid_dep_keys(invalid_dependencies)
),
)),
# Should be the same as 'rsc-and-zinc' case
self.JvmCompileWorkflowType.outline_and_zinc:
lambda: zinc_jobs.append(
make_zinc_job(
compile_target,
input_product_key="rsc_mixed_compile_classpath",
output_products=[runtime_classpath_product],
dep_keys=list(
all_zinc_rsc_invalid_dep_keys(invalid_dependencies)
),
)),
},
)()
compile_jobs = rsc_jobs + zinc_jobs
# Create a job that depends on all real work having completed that will eagerly write to the
# cache by calling `vt.update()`.
write_to_cache_job = Job(
key=self._write_to_cache_key_for_target(compile_target),
fn=ivts.update,
dependencies=[job.key for job in compile_jobs],
run_asap=True,
on_failure=ivts.force_invalidate,
options_scope=self.options_scope,
target=compile_target,
)
all_jobs = cache_doublecheck_jobs + rsc_jobs + zinc_jobs + [
write_to_cache_job
]
return (all_jobs, len(compile_jobs))
def pre_compile_jobs(self, counter):
# Create a target for the jdk outlining so that it'll only be done once per run.
target = Target('jdk', Address('', 'jdk'), self.context.build_graph)
index_dir = os.path.join(self.versioned_workdir, '--jdk--', 'index')
def work_for_vts_rsc_jdk():
distribution = self._get_jvm_distribution()
jvm_lib_jars_abs = distribution.find_libs(['rt.jar', 'dt.jar', 'jce.jar', 'tools.jar'])
self._jvm_lib_jars_abs = jvm_lib_jars_abs
metacp_inputs = tuple(jvm_lib_jars_abs)
counter_val = str(counter()).rjust(counter.format_length(), ' ' if PY3 else b' ')
counter_str = '[{}/{}] '.format(counter_val, counter.size)
self.context.log.info(
counter_str,
'Metacp-ing ',
items_to_report_element(metacp_inputs, 'jar'),
' in the jdk')
# NB: Metacp doesn't handle the existence of possibly stale semanticdb jars,
# so we explicitly clean the directory to keep it happy.
safe_mkdir(index_dir, clean=True)
with Timer() as timer:
# Step 1: Convert classpath to SemanticDB
# ---------------------------------------
rsc_index_dir = fast_relpath(index_dir, get_buildroot())
args = [
'--verbose',
# NB: The directory to dump the semanticdb jars generated by metacp.
'--out', rsc_index_dir,
os.pathsep.join(metacp_inputs),
]
metacp_wu = self._runtool(
'scala.meta.cli.Metacp',
'metacp',
args,
distribution,
tgt=target,
input_files=tuple(
# NB: no input files because the jdk is expected to exist on the system in a known
# location.
# Related: https://github.com/pantsbuild/pants/issues/6416
),
output_dir=rsc_index_dir)
metacp_stdout = stdout_contents(metacp_wu)
metacp_result = json.loads(metacp_stdout)
metai_classpath = self._collect_metai_classpath(metacp_result, jvm_lib_jars_abs)
# Step 1.5: metai Index the semanticdbs
# -------------------------------------
self._run_metai_tool(distribution, metai_classpath, rsc_index_dir, tgt=target)
self._jvm_lib_metacp_classpath = [os.path.join(get_buildroot(), x) for x in metai_classpath]
self._record_target_stats(target,
len(self._jvm_lib_metacp_classpath),
len([]),
timer.elapsed,
False,
'metacp'
)
return [
Job(
'metacp(jdk)',
functools.partial(
work_for_vts_rsc_jdk
),
[],
self._size_estimator([]),
),
]
def create_compile_jobs(self, compile_target, compile_contexts,
invalid_dependencies, ivts, counter,
runtime_classpath_product):
def work_for_vts_rsc(vts, ctx):
target = ctx.target
tgt, = vts.targets
# If we didn't hit the cache in the cache job, run rsc.
if not vts.valid:
counter_val = str(counter()).rjust(counter.format_length(),
' ')
counter_str = '[{}/{}] '.format(counter_val, counter.size)
self.context.log.info(
counter_str, 'Rsc-ing ',
items_to_report_element(ctx.sources,
'{} source'.format(self.name())),
' in ',
items_to_report_element(
[t.address.reference() for t in vts.targets],
'target'), ' (', ctx.target.address.spec, ').')
# This does the following
# - Collect the rsc classpath elements, including zinc compiles of rsc incompatible targets
# and rsc compiles of rsc compatible targets.
# - Run Rsc on the current target with those as dependencies.
dependencies_for_target = list(
DependencyContext.global_instance(
).dependencies_respecting_strict_deps(target))
classpath_paths = []
classpath_directory_digests = []
classpath_product = self.context.products.get_data(
'rsc_mixed_compile_classpath')
classpath_entries = classpath_product.get_classpath_entries_for_targets(
dependencies_for_target)
for _conf, classpath_entry in classpath_entries:
classpath_paths.append(
fast_relpath(classpath_entry.path, get_buildroot()))
if self.HERMETIC == self.execution_strategy_enum.value and not classpath_entry.directory_digest:
raise AssertionError(
"ClasspathEntry {} didn't have a Digest, so won't be present for hermetic "
"execution of rsc".format(classpath_entry))
classpath_directory_digests.append(
classpath_entry.directory_digest)
ctx.ensure_output_dirs_exist()
with Timer() as timer:
# Outline Scala sources into SemanticDB / scalac compatible header jars.
# ---------------------------------------------
rsc_jar_file_relative_path = fast_relpath(
ctx.rsc_jar_file.path, get_buildroot())
sources_snapshot = ctx.target.sources_snapshot(
scheduler=self.context._scheduler)
distribution = self._get_jvm_distribution()
def hermetic_digest_classpath():
jdk_libs_rel, jdk_libs_digest = self._jdk_libs_paths_and_digest(
distribution)
merged_sources_and_jdk_digest = self.context._scheduler.merge_directories(
(jdk_libs_digest,
sources_snapshot.directory_digest) +
tuple(classpath_directory_digests))
classpath_rel_jdk = classpath_paths + jdk_libs_rel
return (merged_sources_and_jdk_digest,
classpath_rel_jdk)
def nonhermetic_digest_classpath():
classpath_abs_jdk = classpath_paths + self._jdk_libs_abs(
distribution)
return ((EMPTY_DIRECTORY_DIGEST), classpath_abs_jdk)
(input_digest, classpath_entry_paths
) = self.execution_strategy_enum.resolve_for_enum_variant(
{
self.HERMETIC: hermetic_digest_classpath,
self.SUBPROCESS: nonhermetic_digest_classpath,
self.NAILGUN: nonhermetic_digest_classpath,
})()
target_sources = ctx.sources
args = [
'-cp',
os.pathsep.join(classpath_entry_paths),
'-d',
rsc_jar_file_relative_path,
] + self.get_options().extra_rsc_args + target_sources
self.write_argsfile(ctx, args)
self._runtool(distribution, input_digest, ctx)
self._record_target_stats(tgt, len(classpath_entry_paths),
len(target_sources), timer.elapsed,
False, 'rsc')
# Update the products with the latest classes.
self.register_extra_products_from_contexts([ctx.target],
compile_contexts)
### Create Jobs for ExecutionGraph
cache_doublecheck_jobs = []
rsc_jobs = []
zinc_jobs = []
# Invalidated targets are a subset of relevant targets: get the context for this one.
compile_target = ivts.target
merged_compile_context = compile_contexts[compile_target]
rsc_compile_context = merged_compile_context.rsc_cc
zinc_compile_context = merged_compile_context.zinc_cc
cache_doublecheck_key = self.exec_graph_double_check_cache_key_for_target(
compile_target)
def all_zinc_rsc_invalid_dep_keys(invalid_deps):
"""Get the rsc key for an rsc-and-zinc target, or the zinc key for a zinc-only target."""
for tgt in invalid_deps:
# None can occur for e.g. JarLibrary deps, which we don't need to compile as they are
# populated in the resolve goal.
tgt_rsc_cc = compile_contexts[tgt].rsc_cc
if tgt_rsc_cc.workflow is not None:
# Rely on the results of zinc compiles for zinc-compatible targets
yield self._key_for_target_as_dep(tgt, tgt_rsc_cc.workflow)
def make_cache_doublecheck_job(dep_keys):
# As in JvmCompile.create_compile_jobs, we create a cache-double-check job that all "real" work
# depends on. It depends on completion of the same dependencies as the rsc job in order to run
# as late as possible, while still running before rsc or zinc.
return Job(cache_doublecheck_key,
functools.partial(
self._default_double_check_cache_for_vts, ivts),
dependencies=list(dep_keys))
def make_rsc_job(target, dep_targets):
return Job(
key=self._rsc_key_for_target(target),
fn=functools.partial(
# NB: This will output to the 'rsc_mixed_compile_classpath' product via
# self.register_extra_products_from_contexts()!
work_for_vts_rsc,
ivts,
rsc_compile_context,
),
# The rsc jobs depend on other rsc jobs, and on zinc jobs for targets that are not
# processed by rsc.
dependencies=[cache_doublecheck_key] +
list(all_zinc_rsc_invalid_dep_keys(dep_targets)),
size=self._size_estimator(rsc_compile_context.sources),
)
def only_zinc_invalid_dep_keys(invalid_deps):
for tgt in invalid_deps:
rsc_cc_tgt = compile_contexts[tgt].rsc_cc
if rsc_cc_tgt.workflow is not None:
yield self._zinc_key_for_target(tgt, rsc_cc_tgt.workflow)
def make_zinc_job(target, input_product_key, output_products,
dep_keys):
return Job(
key=self._zinc_key_for_target(target,
rsc_compile_context.workflow),
fn=functools.partial(self._default_work_for_vts, ivts,
zinc_compile_context, input_product_key,
counter, compile_contexts,
CompositeProductAdder(*output_products)),
dependencies=[cache_doublecheck_key] + list(dep_keys),
size=self._size_estimator(zinc_compile_context.sources),
)
workflow = rsc_compile_context.workflow
# Replica of JvmCompile's _record_target_stats logic
def record(k, v):
self.context.run_tracker.report_target_info(
self.options_scope, compile_target, ['compile', k], v)
record('workflow', workflow.value)
record('execution_strategy', self.execution_strategy)
# Create the cache doublecheck job.
workflow.resolve_for_enum_variant({
'zinc-only':
lambda: cache_doublecheck_jobs.append(
make_cache_doublecheck_job(
list(all_zinc_rsc_invalid_dep_keys(invalid_dependencies)))
),
'zinc-java':
lambda: cache_doublecheck_jobs.append(
make_cache_doublecheck_job(
list(only_zinc_invalid_dep_keys(invalid_dependencies)))),
'rsc-and-zinc':
lambda: cache_doublecheck_jobs.append(
make_cache_doublecheck_job(
list(all_zinc_rsc_invalid_dep_keys(invalid_dependencies)))
),
})()
# Create the rsc job.
# Currently, rsc only supports outlining scala.
workflow.resolve_for_enum_variant({
'zinc-only':
lambda: None,
'zinc-java':
lambda: None,
'rsc-and-zinc':
lambda: rsc_jobs.append(
make_rsc_job(compile_target, invalid_dependencies)),
})()
# Create the zinc compile jobs.
# - Scala zinc compile jobs depend on the results of running rsc on the scala target.
# - Java zinc compile jobs depend on the zinc compiles of their dependencies, because we can't
# generate jars that make javac happy at this point.
workflow.resolve_for_enum_variant({
# NB: zinc-only zinc jobs run zinc and depend on rsc and/or zinc compile outputs.
'zinc-only':
lambda: zinc_jobs.append(
make_zinc_job(compile_target,
input_product_key='rsc_mixed_compile_classpath',
output_products=[
runtime_classpath_product,
self.context.products.get_data(
'rsc_mixed_compile_classpath'),
],
dep_keys=list(
all_zinc_rsc_invalid_dep_keys(
invalid_dependencies)))),
# NB: javac can't read rsc output yet, so we need it to depend strictly on zinc
# compilations of dependencies.
'zinc-java':
lambda: zinc_jobs.append(
make_zinc_job(compile_target,
input_product_key='runtime_classpath',
output_products=[
runtime_classpath_product,
self.context.products.get_data(
'rsc_mixed_compile_classpath'),
],
dep_keys=list(
only_zinc_invalid_dep_keys(
invalid_dependencies)))),
'rsc-and-zinc':
lambda: zinc_jobs.append(
# NB: rsc-and-zinc jobs run zinc and depend on both rsc and zinc compile outputs.
make_zinc_job(
compile_target,
input_product_key='rsc_mixed_compile_classpath',
# NB: We want to ensure the 'runtime_classpath' product *only* contains the outputs of
# zinc compiles, and that the 'rsc_mixed_compile_classpath' entries for rsc-compatible targets
# *only* contain the output of an rsc compile for that target.
output_products=[
runtime_classpath_product,
],
dep_keys=list(
all_zinc_rsc_invalid_dep_keys(invalid_dependencies)),
)),
})()
compile_jobs = rsc_jobs + zinc_jobs
# Create a job that depends on all real work having completed that will eagerly write to the
# cache by calling `vt.update()`.
write_to_cache_job = Job(
key=self._write_to_cache_key_for_target(compile_target),
fn=ivts.update,
dependencies=[job.key for job in compile_jobs],
run_asap=True,
on_failure=ivts.force_invalidate)
all_jobs = cache_doublecheck_jobs + rsc_jobs + zinc_jobs + [
write_to_cache_job
]
return (all_jobs, len(compile_jobs))
def create_compile_jobs(self,
compile_target,
compile_contexts,
invalid_dependencies,
ivts,
counter,
runtime_classpath_product):
def work_for_vts_rsc(vts, ctx):
# Double check the cache before beginning compilation
hit_cache = self.check_cache(vts, counter)
target = ctx.target
tgt, = vts.targets
if not hit_cache:
counter_val = str(counter()).rjust(counter.format_length(), ' ' if PY3 else b' ')
counter_str = '[{}/{}] '.format(counter_val, counter.size)
self.context.log.info(
counter_str,
'Rsc-ing ',
items_to_report_element(ctx.sources, '{} source'.format(self.name())),
' in ',
items_to_report_element([t.address.reference() for t in vts.targets], 'target'),
' (',
ctx.target.address.spec,
').')
# This does the following
# - collect jar dependencies and metacp-classpath entries for them
# - collect the non-java targets and their classpath entries
# - break out java targets and their javac'd classpath entries
# metacp
# - metacp the java targets
# rsc
# - combine the metacp outputs for jars, previous scala targets and the java metacp
# classpath
# - run Rsc on the current target with those as dependencies
dependencies_for_target = list(
DependencyContext.global_instance().dependencies_respecting_strict_deps(target))
jar_deps = [t for t in dependencies_for_target if isinstance(t, JarLibrary)]
def is_java_compile_target(t):
return isinstance(t, JavaLibrary) or t.has_sources('.java')
java_deps = [t for t in dependencies_for_target
if is_java_compile_target(t)]
non_java_deps = [t for t in dependencies_for_target
if not (is_java_compile_target(t)) and not isinstance(t, JarLibrary)]
metacped_jar_classpath_abs = _paths_from_classpath(
self._metacp_jars_classpath_product.get_for_targets(jar_deps + java_deps)
)
metacped_jar_classpath_abs.extend(self._jvm_lib_metacp_classpath)
metacped_jar_classpath_rel = fast_relpath_collection(metacped_jar_classpath_abs)
non_java_paths = _paths_from_classpath(
self.context.products.get_data('rsc_classpath').get_for_targets(non_java_deps),
collection_type=set)
non_java_rel = fast_relpath_collection(non_java_paths)
ctx.ensure_output_dirs_exist()
distribution = self._get_jvm_distribution()
with Timer() as timer:
# Outline Scala sources into SemanticDB
# ---------------------------------------------
rsc_mjar_file = fast_relpath(ctx.rsc_mjar_file, get_buildroot())
# TODO remove non-rsc entries from non_java_rel in a better way
rsc_semanticdb_classpath = metacped_jar_classpath_rel + \
[j for j in non_java_rel if 'compile/rsc/' in j]
target_sources = ctx.sources
args = [
'-cp', os.pathsep.join(rsc_semanticdb_classpath),
'-d', rsc_mjar_file,
] + target_sources
sources_snapshot = ctx.target.sources_snapshot(scheduler=self.context._scheduler)
self._runtool(
'rsc.cli.Main',
'rsc',
args,
distribution,
tgt=tgt,
input_files=tuple(rsc_semanticdb_classpath),
input_digest=sources_snapshot.directory_digest,
output_dir=os.path.dirname(rsc_mjar_file))
self._record_target_stats(tgt,
len(rsc_semanticdb_classpath),
len(target_sources),
timer.elapsed,
False,
'rsc'
)
# Write any additional resources for this target to the target workdir.
self.write_extra_resources(ctx)
# Update the products with the latest classes.
self.register_extra_products_from_contexts([ctx.target], compile_contexts)
def work_for_vts_metacp(vts, ctx, classpath_product_key):
metacp_dependencies_entries = self._zinc.compile_classpath_entries(
classpath_product_key,
ctx.target,
extra_cp_entries=self._extra_compile_time_classpath)
metacp_dependencies = fast_relpath_collection(c.path for c in metacp_dependencies_entries)
metacp_dependencies_digests = [c.directory_digest for c in metacp_dependencies_entries
if c.directory_digest]
metacp_dependencies_paths_without_digests = fast_relpath_collection(
c.path for c in metacp_dependencies_entries if not c.directory_digest)
classpath_entries = [
cp_entry for (conf, cp_entry) in
self.context.products.get_data(classpath_product_key).get_classpath_entries_for_targets(
[ctx.target])
]
classpath_digests = [c.directory_digest for c in classpath_entries if c.directory_digest]
classpath_paths_without_digests = fast_relpath_collection(
c.path for c in classpath_entries if not c.directory_digest)
classpath_abs = [c.path for c in classpath_entries]
classpath_rel = fast_relpath_collection(classpath_abs)
metacp_inputs = []
metacp_inputs.extend(classpath_rel)
counter_val = str(counter()).rjust(counter.format_length(), ' ' if PY3 else b' ')
counter_str = '[{}/{}] '.format(counter_val, counter.size)
self.context.log.info(
counter_str,
'Metacp-ing ',
items_to_report_element(metacp_inputs, 'jar'),
' in ',
items_to_report_element([t.address.reference() for t in vts.targets], 'target'),
' (',
ctx.target.address.spec,
').')
ctx.ensure_output_dirs_exist()
tgt, = vts.targets
with Timer() as timer:
# Step 1: Convert classpath to SemanticDB
# ---------------------------------------
rsc_index_dir = fast_relpath(ctx.rsc_index_dir, get_buildroot())
args = [
'--verbose',
'--stub-broken-signatures',
'--dependency-classpath', os.pathsep.join(
metacp_dependencies +
fast_relpath_collection(self._jvm_lib_jars_abs)
),
# NB: The directory to dump the semanticdb jars generated by metacp.
'--out', rsc_index_dir,
os.pathsep.join(metacp_inputs),
]
# NB: If we're building a scala library jar,
# also request that metacp generate the indices
# for the scala synthetics.
if self._is_scala_core_library(tgt):
args = [
'--include-scala-library-synthetics',
] + args
distribution = self._get_jvm_distribution()
input_digest = self.context._scheduler.merge_directories(
tuple(classpath_digests + metacp_dependencies_digests))
metacp_wu = self._runtool(
'scala.meta.cli.Metacp',
'metacp',
args,
distribution,
tgt=tgt,
input_digest=input_digest,
input_files=tuple(classpath_paths_without_digests +
metacp_dependencies_paths_without_digests),
output_dir=rsc_index_dir)
metacp_result = json.loads(stdout_contents(metacp_wu))
metai_classpath = self._collect_metai_classpath(metacp_result, classpath_rel)
# Step 1.5: metai Index the semanticdbs
# -------------------------------------
self._run_metai_tool(distribution, metai_classpath, rsc_index_dir, tgt)
abs_output = [(conf, os.path.join(get_buildroot(), x))
for conf in self._confs for x in metai_classpath]
self._metacp_jars_classpath_product.add_for_target(
ctx.target,
abs_output,
)
self._record_target_stats(tgt,
len(abs_output),
len([]),
timer.elapsed,
False,
'metacp'
)
rsc_jobs = []
zinc_jobs = []
# Invalidated targets are a subset of relevant targets: get the context for this one.
compile_target = ivts.target
compile_context_pair = compile_contexts[compile_target]
# Create the rsc job.
# Currently, rsc only supports outlining scala.
if self._only_zinc_compilable(compile_target):
pass
elif self._rsc_compilable(compile_target):
rsc_key = self._rsc_key_for_target(compile_target)
rsc_jobs.append(
Job(
rsc_key,
functools.partial(
work_for_vts_rsc,
ivts,
compile_context_pair[0]),
[self._rsc_key_for_target(target) for target in invalid_dependencies] + ['metacp(jdk)'],
self._size_estimator(compile_context_pair[0].sources),
)
)
elif self._metacpable(compile_target):
rsc_key = self._rsc_key_for_target(compile_target)
rsc_jobs.append(
Job(
rsc_key,
functools.partial(
work_for_vts_metacp,
ivts,
compile_context_pair[0],
'compile_classpath'),
[self._rsc_key_for_target(target) for target in invalid_dependencies] + ['metacp(jdk)'],
self._size_estimator(compile_context_pair[0].sources),
on_success=ivts.update,
on_failure=ivts.force_invalidate,
)
)
else:
raise TaskError("Unexpected target for rsc compile {} with type {}"
.format(compile_target, type(compile_target)))
# Create the zinc compile jobs.
# - Scala zinc compile jobs depend on the results of running rsc on the scala target.
# - Java zinc compile jobs depend on the zinc compiles of their dependencies, because we can't
# generate mjars that make javac happy at this point.
invalid_dependencies_without_jar_metacps = [t for t in invalid_dependencies
if not self._metacpable(t)]
if self._rsc_compilable(compile_target):
full_key = self._compile_against_rsc_key_for_target(compile_target)
zinc_jobs.append(
Job(
full_key,
functools.partial(
self._default_work_for_vts,
ivts,
compile_context_pair[1],
'rsc_classpath',
counter,
compile_contexts,
runtime_classpath_product),
[
self._rsc_key_for_target(compile_target)
] + [
self._rsc_key_for_target(target)
for target in invalid_dependencies_without_jar_metacps
] + [
'metacp(jdk)'
],
self._size_estimator(compile_context_pair[1].sources),
# NB: right now, only the last job will write to the cache, because we don't
# do multiple cache entries per target-task tuple.
on_success=ivts.update,
on_failure=ivts.force_invalidate,
)
)
elif self._only_zinc_compilable(compile_target):
# write to both rsc classpath and runtime classpath
class CompositeProductAdder(object):
def __init__(self, runtime_classpath_product, rsc_classpath_product):
self.rsc_classpath_product = rsc_classpath_product
self.runtime_classpath_product = runtime_classpath_product
def add_for_target(self, *args, **kwargs):
self.runtime_classpath_product.add_for_target(*args, **kwargs)
self.rsc_classpath_product.add_for_target(*args, **kwargs)
zinc_key = self._compile_against_rsc_key_for_target(compile_target)
zinc_jobs.append(
Job(
zinc_key,
functools.partial(
self._default_work_for_vts,
ivts,
compile_context_pair[1],
'runtime_classpath',
counter,
compile_contexts,
CompositeProductAdder(
runtime_classpath_product,
self.context.products.get_data('rsc_classpath'))),
[
self._compile_against_rsc_key_for_target(target)
for target in invalid_dependencies_without_jar_metacps],
self._size_estimator(compile_context_pair[1].sources),
on_failure=ivts.force_invalidate,
)
)
metacp_key = self._metacp_key_for_target(compile_target)
rsc_jobs.append(
Job(
metacp_key,
functools.partial(
work_for_vts_metacp,
ivts,
compile_context_pair[0],
'runtime_classpath'),
[self._metacp_dep_key_for_target(target) for target in invalid_dependencies] + [
'metacp(jdk)',
zinc_key,
],
self._size_estimator(compile_context_pair[0].sources),
on_success=ivts.update,
on_failure=ivts.force_invalidate,
)
)
return rsc_jobs + zinc_jobs