def ping(self, netloc): """Time a single roundtrip to the netloc. :param netloc: string of "host:port" :returns: the fastest ping time for a given netloc and number of tries. or Pinger.UNREACHABLE if ping times out. :rtype: float Note that we don't use actual ICMP pings, because cmd-line ping is inflexible and platform-dependent, so shelling out to it is annoying, and the ICMP python lib can only be called by the superuser. """ netloc_info = (netloc, self._timeout, self._tries) if netloc_info in _global_pinger_memo: return _global_pinger_memo[netloc_info] host, colon, portstr = netloc.partition(':') port = int(portstr) if portstr else None rt_secs = Pinger.UNREACHABLE for _ in range(self._tries): try: with Timer() as timer: conn = httplib.HTTPConnection(host, port, timeout=self._timeout) conn.request( 'HEAD', '/') # Doesn't actually matter if this exists. conn.getresponse() new_rt_secs = timer.elapsed except Exception: new_rt_secs = Pinger.UNREACHABLE rt_secs = min(rt_secs, new_rt_secs) _global_pinger_memo[netloc_info] = rt_secs return rt_secs
def _time(self, work, msg): with Timer() as timer: ret = work() elapsed = timer.elapsed print('%s in %f seconds.' % (msg, elapsed)) self.total_time += elapsed return ret
def test_timer(self): class FakeClock(object): def __init__(self): self._time = 0.0 def time(self): ret = self._time self._time += 0.0001 # Force a little time to elapse. return ret def sleep(self, duration): self._time += duration clock = FakeClock() # Note: to test with the real system clock, use this instead: # import time # clock = time with Timer(clock=clock) as t: self.assertLess(t.start, clock.time()) self.assertGreater(t.elapsed, 0) clock.sleep(0.1) self.assertGreater(t.elapsed, 0.1) clock.sleep(0.1) self.assertTrue(t.finish is None) self.assertGreater(t.elapsed, 0.2) self.assertLess(t.finish, clock.time())
def _default_work_for_vts(self, vts, ctx, input_classpath_product_key, counter, all_compile_contexts, output_classpath_product): progress_message = ctx.target.address.spec # Double check the cache before beginning compilation hit_cache = self.check_cache(vts, counter) if not hit_cache: # Compute the compile classpath for this target. dependency_cp_entries = self._zinc.compile_classpath_entries( input_classpath_product_key, ctx.target, extra_cp_entries=self._extra_compile_time_classpath, ) upstream_analysis = dict( self._upstream_analysis(all_compile_contexts, dependency_cp_entries)) is_incremental = self.should_compile_incrementally(vts, ctx) if not is_incremental: # Purge existing analysis file in non-incremental mode. safe_delete(ctx.analysis_file) # Work around https://github.com/pantsbuild/pants/issues/3670 safe_rmtree(ctx.classes_dir.path) dep_context = DependencyContext.global_instance() tgt, = vts.targets compiler_option_sets = dep_context.defaulted_property( tgt, 'compiler_option_sets') zinc_file_manager = dep_context.defaulted_property( tgt, 'zinc_file_manager') with Timer() as timer: directory_digest = self._compile_vts( vts, ctx, upstream_analysis, dependency_cp_entries, progress_message, tgt.platform, compiler_option_sets, zinc_file_manager, counter) ctx.classes_dir = ClasspathEntry(ctx.classes_dir.path, directory_digest) self._record_target_stats(tgt, len(dependency_cp_entries), len(ctx.sources), timer.elapsed, is_incremental, 'compile') # Write any additional resources for this target to the target workdir. self.write_extra_resources(ctx) # Jar the compiled output. self._create_context_jar(ctx) # Update the products with the latest classes. output_classpath_product.add_for_target( ctx.target, [(conf, self._classpath_for_context(ctx)) for conf in self._confs], ) self.register_extra_products_from_contexts([ctx.target], all_compile_contexts)
def work_for_vts_rsc_jdk(): distribution = self._get_jvm_distribution() jvm_lib_jars_abs = distribution.find_libs(['rt.jar', 'dt.jar', 'jce.jar', 'tools.jar']) self._jvm_lib_jars_abs = jvm_lib_jars_abs metacp_inputs = tuple(jvm_lib_jars_abs) counter_val = str(counter()).rjust(counter.format_length(), ' ' if PY3 else b' ') counter_str = '[{}/{}] '.format(counter_val, counter.size) self.context.log.info( counter_str, 'Metacp-ing ', items_to_report_element(metacp_inputs, 'jar'), ' in the jdk') # NB: Metacp doesn't handle the existence of possibly stale semanticdb jars, # so we explicitly clean the directory to keep it happy. safe_mkdir(index_dir, clean=True) with Timer() as timer: # Step 1: Convert classpath to SemanticDB # --------------------------------------- rsc_index_dir = fast_relpath(index_dir, get_buildroot()) args = [ '--verbose', # NB: The directory to dump the semanticdb jars generated by metacp. '--out', rsc_index_dir, os.pathsep.join(metacp_inputs), ] metacp_wu = self._runtool( 'scala.meta.cli.Metacp', 'metacp', args, distribution, tgt=target, input_files=tuple( # NB: no input files because the jdk is expected to exist on the system in a known # location. # Related: https://github.com/pantsbuild/pants/issues/6416 ), output_dir=rsc_index_dir) metacp_stdout = stdout_contents(metacp_wu) metacp_result = json.loads(metacp_stdout) metai_classpath = self._collect_metai_classpath(metacp_result, jvm_lib_jars_abs) # Step 1.5: metai Index the semanticdbs # ------------------------------------- self._run_metai_tool(distribution, metai_classpath, rsc_index_dir, tgt=target) self._jvm_lib_metacp_classpath = [os.path.join(get_buildroot(), x) for x in metai_classpath] self._record_target_stats(target, len(self._jvm_lib_metacp_classpath), len([]), timer.elapsed, False, 'metacp' )
def _try_ping(cls, url, timeout): try: with Timer() as timer: # We just want to see if we can get the headers. requests.head(url, timeout=timeout) return timer.elapsed except Exception: return Pinger.UNREACHABLE
def worker(worker_key, work): status_table.mark_as(RUNNING, worker_key) try: with Timer() as timer: work() result = (worker_key, SUCCESSFUL, None, timer.elapsed) except BaseException: _, exc_value, exc_traceback = sys.exc_info() result = (worker_key, FAILED, (exc_value, traceback.format_tb(exc_traceback)), timer.elapsed) finished_queue.put(result) jobs_in_flight.decrement()
def _default_work_for_vts(self, vts, ctx, input_classpath_product_key, counter, all_compile_contexts, output_classpath_product): progress_message = ctx.target.address.spec # See whether the cache-doublecheck job hit the cache: if so, noop: otherwise, compile. if vts.valid: counter() else: # Compute the compile classpath for this target. dependency_cp_entries = self._zinc.compile_classpath_entries( input_classpath_product_key, ctx.target, extra_cp_entries=self._extra_compile_time_classpath, ) upstream_analysis = dict( self._upstream_analysis(all_compile_contexts, dependency_cp_entries)) is_incremental = self.should_compile_incrementally(vts, ctx) if not is_incremental: # Purge existing analysis file in non-incremental mode. safe_delete(ctx.analysis_file) # Work around https://github.com/pantsbuild/pants/issues/3670 safe_rmtree(ctx.classes_dir.path) dep_context = DependencyContext.global_instance() tgt, = vts.targets compiler_option_sets = dep_context.defaulted_property( tgt, 'compiler_option_sets') zinc_file_manager = dep_context.defaulted_property( tgt, 'zinc_file_manager') with Timer() as timer: directory_digest = self._compile_vts( vts, ctx, upstream_analysis, dependency_cp_entries, progress_message, tgt.platform, compiler_option_sets, zinc_file_manager, counter) # Store the produced Digest (if any). self._set_directory_digest_for_compile_context( ctx, directory_digest) self._record_target_stats(tgt, len(dependency_cp_entries), len(ctx.sources), timer.elapsed, is_incremental, 'compile') # Update the products with the latest classes. output_classpath_product.add_for_target( ctx.target, [(conf, self._classpath_for_context(ctx)) for conf in self._confs], ) self.register_extra_products_from_contexts([ctx.target], all_compile_contexts)
def work_for_vts(vts, ctx): progress_message = ctx.target.address.spec # Double check the cache before beginning compilation hit_cache = check_cache(vts) if not hit_cache: # Compute the compile classpath for this target. cp_entries = [ctx.classes_dir] cp_entries.extend( self._zinc.compile_classpath( classpath_product_key, ctx.target, extra_cp_entries=self._extra_compile_time_classpath, zinc_compile_instance=self)) upstream_analysis = dict( self._upstream_analysis(compile_contexts, cp_entries)) is_incremental = should_compile_incrementally(vts, ctx) if not is_incremental: # Purge existing analysis file in non-incremental mode. safe_delete(ctx.analysis_file) # Work around https://github.com/pantsbuild/pants/issues/3670 safe_rmtree(ctx.classes_dir) dep_context = DependencyContext.global_instance() tgt, = vts.targets fatal_warnings = dep_context.defaulted_property( tgt, lambda x: x.fatal_warnings) zinc_file_manager = dep_context.defaulted_property( tgt, lambda x: x.zinc_file_manager) with Timer() as timer: self._compile_vts(vts, ctx.target, ctx.sources, ctx.analysis_file, upstream_analysis, cp_entries, ctx.classes_dir, ctx.log_dir, ctx.zinc_args_file, progress_message, tgt.platform, fatal_warnings, zinc_file_manager, counter) self._record_target_stats(tgt, len(cp_entries), len(ctx.sources), timer.elapsed, is_incremental) # Write any additional resources for this target to the target workdir. self.write_extra_resources(ctx) # Jar the compiled output. self._create_context_jar(ctx) # Update the products with the latest classes. self._register_vts([ctx])
def work_for_vts_rsc(vts, ctx): target = ctx.target tgt, = vts.targets # If we didn't hit the cache in the cache job, run rsc. if not vts.valid: counter_val = str(counter()).rjust(counter.format_length(), ' ') counter_str = '[{}/{}] '.format(counter_val, counter.size) self.context.log.info( counter_str, 'Rsc-ing ', items_to_report_element(ctx.sources, '{} source'.format(self.name())), ' in ', items_to_report_element( [t.address.reference() for t in vts.targets], 'target'), ' (', ctx.target.address.spec, ').') # This does the following # - Collect the rsc classpath elements, including zinc compiles of rsc incompatible targets # and rsc compiles of rsc compatible targets. # - Run Rsc on the current target with those as dependencies. dependencies_for_target = list( DependencyContext.global_instance( ).dependencies_respecting_strict_deps(target)) classpath_paths = [] classpath_directory_digests = [] classpath_product = self.context.products.get_data( 'rsc_mixed_compile_classpath') classpath_entries = classpath_product.get_classpath_entries_for_targets( dependencies_for_target) for _conf, classpath_entry in classpath_entries: classpath_paths.append( fast_relpath(classpath_entry.path, get_buildroot())) if self.HERMETIC == self.execution_strategy_enum.value and not classpath_entry.directory_digest: raise AssertionError( "ClasspathEntry {} didn't have a Digest, so won't be present for hermetic " "execution of rsc".format(classpath_entry)) classpath_directory_digests.append( classpath_entry.directory_digest) ctx.ensure_output_dirs_exist() with Timer() as timer: # Outline Scala sources into SemanticDB / scalac compatible header jars. # --------------------------------------------- rsc_jar_file_relative_path = fast_relpath( ctx.rsc_jar_file.path, get_buildroot()) sources_snapshot = ctx.target.sources_snapshot( scheduler=self.context._scheduler) distribution = self._get_jvm_distribution() def hermetic_digest_classpath(): jdk_libs_rel, jdk_libs_digest = self._jdk_libs_paths_and_digest( distribution) merged_sources_and_jdk_digest = self.context._scheduler.merge_directories( (jdk_libs_digest, sources_snapshot.directory_digest) + tuple(classpath_directory_digests)) classpath_rel_jdk = classpath_paths + jdk_libs_rel return (merged_sources_and_jdk_digest, classpath_rel_jdk) def nonhermetic_digest_classpath(): classpath_abs_jdk = classpath_paths + self._jdk_libs_abs( distribution) return ((EMPTY_DIRECTORY_DIGEST), classpath_abs_jdk) (input_digest, classpath_entry_paths ) = self.execution_strategy_enum.resolve_for_enum_variant( { self.HERMETIC: hermetic_digest_classpath, self.SUBPROCESS: nonhermetic_digest_classpath, self.NAILGUN: nonhermetic_digest_classpath, })() target_sources = ctx.sources args = [ '-cp', os.pathsep.join(classpath_entry_paths), '-d', rsc_jar_file_relative_path, ] + self.get_options().extra_rsc_args + target_sources self.write_argsfile(ctx, args) self._runtool(distribution, input_digest, ctx) self._record_target_stats(tgt, len(classpath_entry_paths), len(target_sources), timer.elapsed, False, 'rsc') # Update the products with the latest classes. self.register_extra_products_from_contexts([ctx.target], compile_contexts)
def work_for_vts_rsc(vts, ctx): # Double check the cache before beginning compilation hit_cache = self.check_cache(vts, counter) target = ctx.target tgt, = vts.targets if not hit_cache: counter_val = str(counter()).rjust(counter.format_length(), b' ') counter_str = '[{}/{}] '.format(counter_val, counter.size) self.context.log.info( counter_str, 'Rsc-ing ', items_to_report_element(ctx.sources, '{} source'.format(self.name())), ' in ', items_to_report_element( [t.address.reference() for t in vts.targets], 'target'), ' (', ctx.target.address.spec, ').') # This does the following # - collect jar dependencies and metacp-classpath entries for them # - collect the non-java targets and their classpath entries # - break out java targets and their javac'd classpath entries # metacp # - metacp the java targets # rsc # - combine the metacp outputs for jars, previous scala targets and the java metacp # classpath # - run Rsc on the current target with those as dependencies dependencies_for_target = list( DependencyContext.global_instance( ).dependencies_respecting_strict_deps(target)) jar_deps = [ t for t in dependencies_for_target if isinstance(t, JarLibrary) ] def is_java_compile_target(t): return isinstance(t, JavaLibrary) or t.has_sources('.java') java_deps = [ t for t in dependencies_for_target if is_java_compile_target(t) ] non_java_deps = [ t for t in dependencies_for_target if not (is_java_compile_target(t)) and not isinstance(t, JarLibrary) ] metacped_jar_classpath_abs = _paths_from_classpath( self._metacp_jars_classpath_product.get_for_targets( jar_deps)) metacped_jar_classpath_abs.extend( self._jvm_lib_metacp_classpath) metacped_jar_classpath_rel = fast_relpath_collection( metacped_jar_classpath_abs) jar_rsc_classpath_paths = _paths_from_classpath( self.context.products.get_data( 'rsc_classpath').get_for_targets(jar_deps), collection_type=set) jar_rsc_classpath_rel = fast_relpath_collection( jar_rsc_classpath_paths) non_java_paths = _paths_from_classpath( self.context.products.get_data( 'rsc_classpath').get_for_targets(non_java_deps), collection_type=set) non_java_rel = fast_relpath_collection(non_java_paths) java_paths = _paths_from_classpath( self.context.products.get_data( 'rsc_classpath').get_for_targets(java_deps), collection_type=set) java_rel = fast_relpath_collection(java_paths) ctx.ensure_output_dirs_exist() distribution = self._get_jvm_distribution() with Timer() as timer: # Step 1: Convert classpath to SemanticDB # --------------------------------------- # If there are any as yet not metacp'd dependencies, metacp them so their indices can # be passed to Rsc. # TODO move these to their own jobs. https://github.com/pantsbuild/pants/issues/6754 # Inputs # - Java dependencies jars metacp_inputs = java_rel # Dependencies # - 3rdparty jars # - non-java, ie scala, dependencies # - jdk snapshotable_metacp_dependencies = list(jar_rsc_classpath_rel) + \ list(non_java_rel) + \ fast_relpath_collection( _paths_from_classpath(self._extra_compile_time_classpath)) metacp_dependencies = snapshotable_metacp_dependencies + self._jvm_lib_jars_abs if metacp_inputs: rsc_index_dir = fast_relpath(ctx.rsc_index_dir, get_buildroot()) args = [ '--verbose', '--stub-broken-signatures', '--dependency-classpath', os.pathsep.join(metacp_dependencies), # NB: The directory to dump the semanticdb jars generated by metacp. '--out', rsc_index_dir, os.pathsep.join(metacp_inputs), ] metacp_wu = self._runtool( 'scala.meta.cli.Metacp', 'metacp', args, distribution, tgt=tgt, input_files=tuple( metacp_inputs + snapshotable_metacp_dependencies), output_dir=rsc_index_dir) metacp_stdout = stdout_contents(metacp_wu) metacp_result = json.loads(metacp_stdout) metacped_java_dependency_rel = self._collect_metai_classpath( metacp_result, java_rel) # Step 1.5: metai Index the semanticdbs # ------------------------------------- self._run_metai_tool(distribution, metacped_java_dependency_rel, rsc_index_dir, tgt) else: # NB: there are no unmetacp'd dependencies metacped_java_dependency_rel = [] # Step 2: Outline Scala sources into SemanticDB # --------------------------------------------- rsc_mjar_file = fast_relpath(ctx.rsc_mjar_file, get_buildroot()) # TODO remove non-rsc entries from non_java_rel in a better way rsc_semanticdb_classpath = metacped_java_dependency_rel + \ metacped_jar_classpath_rel + \ [j for j in non_java_rel if 'compile/rsc/' in j] target_sources = ctx.sources args = [ '-cp', os.pathsep.join(rsc_semanticdb_classpath), '-d', rsc_mjar_file, ] + target_sources sources_snapshot = ctx.target.sources_snapshot( scheduler=self.context._scheduler) self._runtool( 'rsc.cli.Main', 'rsc', args, distribution, tgt=tgt, input_files=tuple(rsc_semanticdb_classpath), input_digest=sources_snapshot.directory_digest, output_dir=os.path.dirname(rsc_mjar_file)) self._record_target_stats(tgt, len(metacp_inputs), len(target_sources), timer.elapsed, False, 'rsc') # Write any additional resources for this target to the target workdir. self.write_extra_resources(ctx) # Update the products with the latest classes. self.register_extra_products_from_contexts([ctx.target], compile_contexts)
def work_for_vts_rsc(vts, ctx): target = ctx.target (tgt, ) = vts.targets rsc_cc = compile_contexts[target].rsc_cc use_youtline = rsc_cc.workflow == self.JvmCompileWorkflowType.outline_and_zinc outliner = "scalac-outliner" if use_youtline else "rsc" if use_youtline and Semver.parse( self._scala_library_version) < Semver.parse("2.12.9"): raise RuntimeError( f"To use scalac's built-in outlining, scala version must be at least 2.12.9, but got {self._scala_library_version}" ) # If we didn't hit the cache in the cache job, run rsc. if not vts.valid: counter_val = str(counter()).rjust(counter.format_length(), " ") counter_str = f"[{counter_val}/{counter.size}] " action_str = "Outlining " if use_youtline else "Rsc-ing " self.context.log.info( counter_str, action_str, items_to_report_element(ctx.sources, f"{self.name()} source"), " in ", items_to_report_element( [t.address.reference() for t in vts.targets], "target"), " (", ctx.target.address.spec, ").", ) # This does the following # - Collect the rsc classpath elements, including zinc compiles of rsc incompatible targets # and rsc compiles of rsc compatible targets. # - Run Rsc on the current target with those as dependencies. dependencies_for_target = list( DependencyContext.global_instance( ).dependencies_respecting_strict_deps(target)) classpath_paths = [] classpath_digests = [] classpath_product = self.context.products.get_data( "rsc_mixed_compile_classpath") classpath_entries = classpath_product.get_classpath_entries_for_targets( dependencies_for_target) hermetic = self.execution_strategy == self.ExecutionStrategy.hermetic for _conf, classpath_entry in classpath_entries: classpath_paths.append( fast_relpath(classpath_entry.path, get_buildroot())) if hermetic and not classpath_entry.directory_digest: raise AssertionError( "ClasspathEntry {} didn't have a Digest, so won't be present for hermetic " "execution of {}".format(classpath_entry, outliner)) classpath_digests.append(classpath_entry.directory_digest) ctx.ensure_output_dirs_exist() with Timer() as timer: # Outline Scala sources into SemanticDB / scalac compatible header jars. # --------------------------------------------- rsc_jar_file_relative_path = fast_relpath( ctx.rsc_jar_file.path, get_buildroot()) sources_snapshot = ctx.target.sources_snapshot( scheduler=self.context._scheduler) distribution = self._get_jvm_distribution() def hermetic_digest_classpath(): jdk_libs_rel, jdk_libs_digest = self._jdk_libs_paths_and_digest( distribution) merged_sources_and_jdk_digest = self.context._scheduler.merge_directories( (jdk_libs_digest, sources_snapshot.digest) + tuple(classpath_digests)) classpath_rel_jdk = classpath_paths + jdk_libs_rel return (merged_sources_and_jdk_digest, classpath_rel_jdk) def nonhermetic_digest_classpath(): classpath_abs_jdk = classpath_paths + self._jdk_libs_abs( distribution) return ((EMPTY_DIGEST), classpath_abs_jdk) (input_digest, classpath_entry_paths) = match( self.execution_strategy, { self.ExecutionStrategy.hermetic: hermetic_digest_classpath, self.ExecutionStrategy.subprocess: nonhermetic_digest_classpath, self.ExecutionStrategy.nailgun: nonhermetic_digest_classpath, }, )() youtline_args = [] if use_youtline: youtline_args = [ "-Youtline", "-Ystop-after:pickler", "-Ypickle-write", rsc_jar_file_relative_path, ] target_sources = ctx.sources # TODO: m.jar digests aren't found, so hermetic will fail. if use_youtline and not hermetic and self.get_options( ).zinc_outline: self._zinc_outline(ctx, classpath_paths, target_sources, youtline_args) else: args = ([ "-cp", os.pathsep.join(classpath_entry_paths), "-d", rsc_jar_file_relative_path, ] + self.get_options().extra_rsc_args + youtline_args + target_sources) self.write_argsfile(ctx, args) self._runtool(distribution, input_digest, ctx, use_youtline) self._record_target_stats( tgt, len(classpath_entry_paths), len(target_sources), timer.elapsed, False, outliner, ) # Update the products with the latest classes. self.register_extra_products_from_contexts([ctx.target], compile_contexts)
def work_for_vts_rsc(vts, ctx): target = ctx.target tgt, = vts.targets rsc_cc = compile_contexts[target].rsc_cc use_youtline = rsc_cc.workflow == self.JvmCompileWorkflowType.outline_and_zinc outliner = 'scalac-outliner' if use_youtline else 'rsc' if use_youtline and Semver.parse( self._scala_library_version) < Semver.parse("2.12.9"): raise RuntimeError( f"To use scalac's built-in outlining, scala version must be at least 2.12.9, but got {self._scala_library_version}" ) # If we didn't hit the cache in the cache job, run rsc. if not vts.valid: counter_val = str(counter()).rjust(counter.format_length(), ' ') counter_str = '[{}/{}] '.format(counter_val, counter.size) action_str = 'Outlining ' if use_youtline else 'Rsc-ing ' self.context.log.info( counter_str, action_str, items_to_report_element(ctx.sources, '{} source'.format(self.name())), ' in ', items_to_report_element( [t.address.reference() for t in vts.targets], 'target'), ' (', ctx.target.address.spec, ').') # This does the following # - Collect the rsc classpath elements, including zinc compiles of rsc incompatible targets # and rsc compiles of rsc compatible targets. # - Run Rsc on the current target with those as dependencies. dependencies_for_target = list( DependencyContext.global_instance( ).dependencies_respecting_strict_deps(target)) classpath_paths = [] classpath_directory_digests = [] classpath_product = self.context.products.get_data( 'rsc_mixed_compile_classpath') classpath_entries = classpath_product.get_classpath_entries_for_targets( dependencies_for_target) for _conf, classpath_entry in classpath_entries: classpath_paths.append( fast_relpath(classpath_entry.path, get_buildroot())) if self.execution_strategy == self.ExecutionStrategy.hermetic and not classpath_entry.directory_digest: raise AssertionError( "ClasspathEntry {} didn't have a Digest, so won't be present for hermetic " "execution of {}".format(classpath_entry, outliner)) classpath_directory_digests.append( classpath_entry.directory_digest) ctx.ensure_output_dirs_exist() with Timer() as timer: # Outline Scala sources into SemanticDB / scalac compatible header jars. # --------------------------------------------- rsc_jar_file_relative_path = fast_relpath( ctx.rsc_jar_file.path, get_buildroot()) sources_snapshot = ctx.target.sources_snapshot( scheduler=self.context._scheduler) distribution = self._get_jvm_distribution() def hermetic_digest_classpath(): jdk_libs_rel, jdk_libs_digest = self._jdk_libs_paths_and_digest( distribution) merged_sources_and_jdk_digest = self.context._scheduler.merge_directories( (jdk_libs_digest, sources_snapshot.directory_digest) + tuple(classpath_directory_digests)) classpath_rel_jdk = classpath_paths + jdk_libs_rel return (merged_sources_and_jdk_digest, classpath_rel_jdk) def nonhermetic_digest_classpath(): classpath_abs_jdk = classpath_paths + self._jdk_libs_abs( distribution) return ((EMPTY_DIRECTORY_DIGEST), classpath_abs_jdk) (input_digest, classpath_entry_paths) = self.execution_strategy.match({ self.ExecutionStrategy.hermetic: hermetic_digest_classpath, self.ExecutionStrategy.subprocess: nonhermetic_digest_classpath, self.ExecutionStrategy.nailgun: nonhermetic_digest_classpath, })() youtline_args = [] if use_youtline: youtline_args = [ "-Youtline", "-Ystop-after:pickler", "-Ypickle-write", rsc_jar_file_relative_path, ] if not self.get_options().allow_public_inference: wartremover_args = [ f"-Xplugin:{self._wartremover_classpath[0]}", "-P:wartremover:traverser:org.wartremover.warts.PublicInference", "-Ycache-plugin-class-loader:last-modified", ] youtline_args = wartremover_args + youtline_args target_sources = ctx.sources args = [ '-cp', os.pathsep.join(classpath_entry_paths), '-d', rsc_jar_file_relative_path, ] + self.get_options( ).extra_rsc_args + youtline_args + target_sources self.write_argsfile(ctx, args) self._runtool(distribution, input_digest, ctx, use_youtline) self._record_target_stats(tgt, len(classpath_entry_paths), len(target_sources), timer.elapsed, False, outliner) # Update the products with the latest classes. self.register_extra_products_from_contexts([ctx.target], compile_contexts)
def work_for_vts_rsc(vts, ctx): # Double check the cache before beginning compilation hit_cache = self.check_cache(vts, counter) target = ctx.target if not hit_cache: cp_entries = [] # Include the current machine's jdk lib jars. This'll blow up remotely. # We need a solution for that. # Probably something to do with https://github.com/pantsbuild/pants/pull/6346 distribution = JvmPlatform.preferred_jvm_distribution([ctx.target.platform], strict=True) jvm_lib_jars_abs = distribution.find_libs(['rt.jar', 'dt.jar', 'jce.jar', 'tools.jar']) cp_entries.extend(jvm_lib_jars_abs) classpath_abs = self._zinc.compile_classpath( 'rsc_classpath', ctx.target, extra_cp_entries=self._extra_compile_time_classpath) jar_deps = [t for t in DependencyContext.global_instance().dependencies_respecting_strict_deps(target) if isinstance(t, JarLibrary)] metacp_jar_classpath_abs = [y[1] for y in self._metacp_jars_classpath_product.get_for_targets( jar_deps )] jar_jar_paths = {y[1] for y in self.context.products.get_data('rsc_classpath').get_for_targets(jar_deps)} classpath_abs = [c for c in classpath_abs if c not in jar_jar_paths] classpath_rel = fast_relpath_collection(classpath_abs) metacp_jar_classpath_rel = fast_relpath_collection(metacp_jar_classpath_abs) cp_entries.extend(classpath_rel) ctx.ensure_output_dirs_exist() counter_val = str(counter()).rjust(counter.format_length(), b' ') counter_str = '[{}/{}] '.format(counter_val, counter.size) self.context.log.info( counter_str, 'Rsc-ing ', items_to_report_element(ctx.sources, '{} source'.format(self.name())), ' in ', items_to_report_element([t.address.reference() for t in vts.targets], 'target'), ' (', ctx.target.address.spec, ').') tgt, = vts.targets with Timer() as timer: # Step 1: Convert classpath to SemanticDB # --------------------------------------- scalac_classpath_path_entries_abs = self.tool_classpath('workaround-metacp-dependency-classpath') scalac_classpath_path_entries = fast_relpath_collection(scalac_classpath_path_entries_abs) rsc_index_dir = fast_relpath(ctx.rsc_index_dir, get_buildroot()) args = [ '--verbose', # NB: Without this setting, rsc will be missing some symbols # from the scala library. '--include-scala-library-synthetics', # TODO generate these once and cache them # NB: We need to add these extra dependencies in order to be able # to find symbols used by the scalac jars. '--dependency-classpath', os.pathsep.join(scalac_classpath_path_entries + list(jar_jar_paths)), # NB: The directory to dump the semanticdb jars generated by metacp. '--out', rsc_index_dir, os.pathsep.join(cp_entries), ] metacp_wu = self._runtool( 'scala.meta.cli.Metacp', 'metacp', args, distribution, tgt=tgt, input_files=(scalac_classpath_path_entries + classpath_rel), output_dir=rsc_index_dir) metacp_stdout = stdout_contents(metacp_wu) metacp_result = json.loads(metacp_stdout) metai_classpath = self._collect_metai_classpath( metacp_result, classpath_rel, jvm_lib_jars_abs) # Step 1.5: metai Index the semanticdbs # ------------------------------------- self._run_metai_tool(distribution, metai_classpath, rsc_index_dir, tgt) # Step 2: Outline Scala sources into SemanticDB # --------------------------------------------- rsc_outline_dir = fast_relpath(ctx.rsc_outline_dir, get_buildroot()) rsc_out = os.path.join(rsc_outline_dir, 'META-INF/semanticdb/out.semanticdb') safe_mkdir(os.path.join(rsc_outline_dir, 'META-INF/semanticdb')) target_sources = ctx.sources args = [ '-cp', os.pathsep.join(metai_classpath + metacp_jar_classpath_rel), '-out', rsc_out, ] + target_sources self._runtool( 'rsc.cli.Main', 'rsc', args, distribution, tgt=tgt, # TODO pass the input files from the target snapshot instead of the below # input_snapshot = ctx.target.sources_snapshot(scheduler=self.context._scheduler) input_files=target_sources + metai_classpath + metacp_jar_classpath_rel, output_dir=rsc_outline_dir) rsc_classpath = [rsc_outline_dir] # Step 2.5: Postprocess the rsc outputs # TODO: This is only necessary as a workaround for https://github.com/twitter/rsc/issues/199. # Ideally, Rsc would do this on its own. self._run_metai_tool(distribution, rsc_classpath, rsc_outline_dir, tgt, extra_input_files=(rsc_out,)) # Step 3: Convert SemanticDB into an mjar # --------------------------------------- rsc_mjar_file = fast_relpath(ctx.rsc_mjar_file, get_buildroot()) args = [ '-out', rsc_mjar_file, os.pathsep.join(rsc_classpath), ] self._runtool( 'scala.meta.cli.Mjar', 'mjar', args, distribution, tgt=tgt, input_files=( rsc_out, ), output_dir=os.path.dirname(rsc_mjar_file) ) self.context.products.get_data('rsc_classpath').add_for_target( ctx.target, [(conf, ctx.rsc_mjar_file) for conf in self._confs], ) self._record_target_stats(tgt, len(cp_entries), len(target_sources), timer.elapsed, False, 'rsc' ) # Write any additional resources for this target to the target workdir. self.write_extra_resources(ctx) # Update the products with the latest classes. self.register_extra_products_from_contexts([ctx.target], compile_contexts)
def work_for_vts_rsc(vts, ctx): # Double check the cache before beginning compilation hit_cache = self.check_cache(vts, counter) target = ctx.target tgt, = vts.targets if not hit_cache: counter_val = str(counter()).rjust(counter.format_length(), ' ' if PY3 else b' ') counter_str = '[{}/{}] '.format(counter_val, counter.size) self.context.log.info( counter_str, 'Rsc-ing ', items_to_report_element(ctx.sources, '{} source'.format(self.name())), ' in ', items_to_report_element([t.address.reference() for t in vts.targets], 'target'), ' (', ctx.target.address.spec, ').') # This does the following # - Collect the rsc classpath elements, including zinc compiles of rsc incompatible targets # and rsc compiles of rsc compatible targets. # - Run Rsc on the current target with those as dependencies. dependencies_for_target = list( DependencyContext.global_instance().dependencies_respecting_strict_deps(target)) rsc_deps_classpath_unprocessed = _paths_from_classpath( self.context.products.get_data('rsc_classpath').get_for_targets(dependencies_for_target), collection_type=OrderedSet) rsc_classpath_rel = fast_relpath_collection(list(rsc_deps_classpath_unprocessed)) ctx.ensure_output_dirs_exist() with Timer() as timer: # Outline Scala sources into SemanticDB / scalac compatible header jars. # --------------------------------------------- rsc_jar_file = fast_relpath(ctx.rsc_jar_file, get_buildroot()) sources_snapshot = ctx.target.sources_snapshot(scheduler=self.context._scheduler) distribution = self._get_jvm_distribution() def hermetic_digest_classpath(): jdk_libs_rel, jdk_libs_digest = self._jdk_libs_paths_and_digest(distribution) merged_sources_and_jdk_digest = self.context._scheduler.merge_directories( (jdk_libs_digest, sources_snapshot.directory_digest)) classpath_rel_jdk = rsc_classpath_rel + jdk_libs_rel return (merged_sources_and_jdk_digest, classpath_rel_jdk) def nonhermetic_digest_classpath(): classpath_abs_jdk = rsc_classpath_rel + self._jdk_libs_abs(distribution) return ((EMPTY_DIRECTORY_DIGEST), classpath_abs_jdk) (input_digest, classpath_entry_paths) = self.execution_strategy_enum.resolve_for_enum_variant({ self.HERMETIC: hermetic_digest_classpath, self.SUBPROCESS: nonhermetic_digest_classpath, self.NAILGUN: nonhermetic_digest_classpath, })() target_sources = ctx.sources args = [ '-cp', os.pathsep.join(classpath_entry_paths), '-d', rsc_jar_file, ] + target_sources self._runtool( 'rsc.cli.Main', 'rsc', args, distribution, tgt=tgt, input_files=tuple(rsc_classpath_rel), input_digest=input_digest, output_dir=os.path.dirname(rsc_jar_file)) self._record_target_stats(tgt, len(rsc_classpath_rel), len(target_sources), timer.elapsed, False, 'rsc' ) # Write any additional resources for this target to the target workdir. self.write_extra_resources(ctx) # Update the products with the latest classes. self.register_extra_products_from_contexts([ctx.target], compile_contexts)
def work_for_vts_metacp(vts, ctx, classpath_product_key): metacp_dependencies_entries = self._zinc.compile_classpath_entries( classpath_product_key, ctx.target, extra_cp_entries=self._extra_compile_time_classpath) metacp_dependencies = fast_relpath_collection(c.path for c in metacp_dependencies_entries) metacp_dependencies_digests = [c.directory_digest for c in metacp_dependencies_entries if c.directory_digest] metacp_dependencies_paths_without_digests = fast_relpath_collection( c.path for c in metacp_dependencies_entries if not c.directory_digest) classpath_entries = [ cp_entry for (conf, cp_entry) in self.context.products.get_data(classpath_product_key).get_classpath_entries_for_targets( [ctx.target]) ] classpath_digests = [c.directory_digest for c in classpath_entries if c.directory_digest] classpath_paths_without_digests = fast_relpath_collection( c.path for c in classpath_entries if not c.directory_digest) classpath_abs = [c.path for c in classpath_entries] classpath_rel = fast_relpath_collection(classpath_abs) metacp_inputs = [] metacp_inputs.extend(classpath_rel) counter_val = str(counter()).rjust(counter.format_length(), ' ' if PY3 else b' ') counter_str = '[{}/{}] '.format(counter_val, counter.size) self.context.log.info( counter_str, 'Metacp-ing ', items_to_report_element(metacp_inputs, 'jar'), ' in ', items_to_report_element([t.address.reference() for t in vts.targets], 'target'), ' (', ctx.target.address.spec, ').') ctx.ensure_output_dirs_exist() tgt, = vts.targets with Timer() as timer: # Step 1: Convert classpath to SemanticDB # --------------------------------------- rsc_index_dir = fast_relpath(ctx.rsc_index_dir, get_buildroot()) args = [ '--verbose', '--stub-broken-signatures', '--dependency-classpath', os.pathsep.join( metacp_dependencies + fast_relpath_collection(self._jvm_lib_jars_abs) ), # NB: The directory to dump the semanticdb jars generated by metacp. '--out', rsc_index_dir, os.pathsep.join(metacp_inputs), ] # NB: If we're building a scala library jar, # also request that metacp generate the indices # for the scala synthetics. if self._is_scala_core_library(tgt): args = [ '--include-scala-library-synthetics', ] + args distribution = self._get_jvm_distribution() input_digest = self.context._scheduler.merge_directories( tuple(classpath_digests + metacp_dependencies_digests)) metacp_wu = self._runtool( 'scala.meta.cli.Metacp', 'metacp', args, distribution, tgt=tgt, input_digest=input_digest, input_files=tuple(classpath_paths_without_digests + metacp_dependencies_paths_without_digests), output_dir=rsc_index_dir) metacp_result = json.loads(stdout_contents(metacp_wu)) metai_classpath = self._collect_metai_classpath(metacp_result, classpath_rel) # Step 1.5: metai Index the semanticdbs # ------------------------------------- self._run_metai_tool(distribution, metai_classpath, rsc_index_dir, tgt) abs_output = [(conf, os.path.join(get_buildroot(), x)) for conf in self._confs for x in metai_classpath] self._metacp_jars_classpath_product.add_for_target( ctx.target, abs_output, ) self._record_target_stats(tgt, len(abs_output), len([]), timer.elapsed, False, 'metacp' )
def work_for_vts_rsc(vts, ctx): # Double check the cache before beginning compilation hit_cache = self.check_cache(vts, counter) target = ctx.target tgt, = vts.targets if not hit_cache: counter_val = str(counter()).rjust(counter.format_length(), ' ' if PY3 else b' ') counter_str = '[{}/{}] '.format(counter_val, counter.size) self.context.log.info( counter_str, 'Rsc-ing ', items_to_report_element(ctx.sources, '{} source'.format(self.name())), ' in ', items_to_report_element([t.address.reference() for t in vts.targets], 'target'), ' (', ctx.target.address.spec, ').') # This does the following # - collect jar dependencies and metacp-classpath entries for them # - collect the non-java targets and their classpath entries # - break out java targets and their javac'd classpath entries # metacp # - metacp the java targets # rsc # - combine the metacp outputs for jars, previous scala targets and the java metacp # classpath # - run Rsc on the current target with those as dependencies dependencies_for_target = list( DependencyContext.global_instance().dependencies_respecting_strict_deps(target)) jar_deps = [t for t in dependencies_for_target if isinstance(t, JarLibrary)] def is_java_compile_target(t): return isinstance(t, JavaLibrary) or t.has_sources('.java') java_deps = [t for t in dependencies_for_target if is_java_compile_target(t)] non_java_deps = [t for t in dependencies_for_target if not (is_java_compile_target(t)) and not isinstance(t, JarLibrary)] metacped_jar_classpath_abs = _paths_from_classpath( self._metacp_jars_classpath_product.get_for_targets(jar_deps + java_deps) ) metacped_jar_classpath_abs.extend(self._jvm_lib_metacp_classpath) metacped_jar_classpath_rel = fast_relpath_collection(metacped_jar_classpath_abs) non_java_paths = _paths_from_classpath( self.context.products.get_data('rsc_classpath').get_for_targets(non_java_deps), collection_type=set) non_java_rel = fast_relpath_collection(non_java_paths) ctx.ensure_output_dirs_exist() distribution = self._get_jvm_distribution() with Timer() as timer: # Outline Scala sources into SemanticDB # --------------------------------------------- rsc_mjar_file = fast_relpath(ctx.rsc_mjar_file, get_buildroot()) # TODO remove non-rsc entries from non_java_rel in a better way rsc_semanticdb_classpath = metacped_jar_classpath_rel + \ [j for j in non_java_rel if 'compile/rsc/' in j] target_sources = ctx.sources args = [ '-cp', os.pathsep.join(rsc_semanticdb_classpath), '-d', rsc_mjar_file, ] + target_sources sources_snapshot = ctx.target.sources_snapshot(scheduler=self.context._scheduler) self._runtool( 'rsc.cli.Main', 'rsc', args, distribution, tgt=tgt, input_files=tuple(rsc_semanticdb_classpath), input_digest=sources_snapshot.directory_digest, output_dir=os.path.dirname(rsc_mjar_file)) self._record_target_stats(tgt, len(rsc_semanticdb_classpath), len(target_sources), timer.elapsed, False, 'rsc' ) # Write any additional resources for this target to the target workdir. self.write_extra_resources(ctx) # Update the products with the latest classes. self.register_extra_products_from_contexts([ctx.target], compile_contexts)
def work_for_vts_rsc_jar_library(vts, ctx): cp_entries = [] # Include the current machine's jdk lib jars. This'll blow up remotely. # We need a solution for that. # Probably something to do with https://github.com/pantsbuild/pants/pull/6346 # TODO perhaps determine the platform of the jar and use that here. # https://github.com/pantsbuild/pants/issues/6547 distribution = JvmPlatform.preferred_jvm_distribution([], strict=True) jvm_lib_jars_abs = distribution.find_libs(['rt.jar', 'dt.jar', 'jce.jar', 'tools.jar']) cp_entries.extend(jvm_lib_jars_abs) # TODO use compile_classpath classpath_abs = [ path for (conf, path) in self.context.products.get_data('rsc_classpath').get_for_target(ctx.target) ] dependency_classpath = self._zinc.compile_classpath( 'compile_classpath', ctx.target, extra_cp_entries=self._extra_compile_time_classpath) classpath_rel = fast_relpath_collection(classpath_abs) cp_entries.extend(classpath_rel) counter_val = str(counter()).rjust(counter.format_length(), b' ') counter_str = '[{}/{}] '.format(counter_val, counter.size) self.context.log.info( counter_str, 'Metacp-ing ', items_to_report_element(cp_entries, 'jar'), ' in ', items_to_report_element([t.address.reference() for t in vts.targets], 'target'), ' (', ctx.target.address.spec, ').') ctx.ensure_output_dirs_exist() tgt, = vts.targets with Timer() as timer: # Step 1: Convert classpath to SemanticDB # --------------------------------------- scalac_classpath_path_entries_abs = self.tool_classpath('workaround-metacp-dependency-classpath') scalac_classpath_path_entries = fast_relpath_collection(scalac_classpath_path_entries_abs) rsc_index_dir = fast_relpath(ctx.rsc_index_dir, get_buildroot()) args = [ '--verbose', # NB: Without this setting, rsc will be missing some symbols # from the scala library. '--include-scala-library-synthetics', # TODO generate these once and cache them # NB: We need to add these extra dependencies in order to be able # to find symbols used by the scalac jars. '--dependency-classpath', os.pathsep.join(dependency_classpath + scalac_classpath_path_entries), # NB: The directory to dump the semanticdb jars generated by metacp. '--out', rsc_index_dir, os.pathsep.join(cp_entries), ] metacp_wu = self._runtool( 'scala.meta.cli.Metacp', 'metacp', args, distribution, tgt=tgt, input_files=(scalac_classpath_path_entries + classpath_rel), output_dir=rsc_index_dir) metacp_stdout = stdout_contents(metacp_wu) metacp_result = json.loads(metacp_stdout) metai_classpath = self._collect_metai_classpath( metacp_result, classpath_rel, jvm_lib_jars_abs) # Step 1.5: metai Index the semanticdbs # ------------------------------------- self._run_metai_tool(distribution, metai_classpath, rsc_index_dir, tgt) abs_output = [(conf, os.path.join(get_buildroot(), x)) for conf in self._confs for x in metai_classpath] self._metacp_jars_classpath_product.add_for_target( ctx.target, abs_output, ) self._record_target_stats(tgt, len(abs_output), len([]), timer.elapsed, False, 'metacp' )
def work_for_vts_rsc_jar_library(vts, ctx): distribution = self._get_jvm_distribution() # TODO use compile_classpath classpath_abs = [ path for (conf, path) in self.context.products.get_data( 'rsc_classpath').get_for_target(ctx.target) ] dependency_classpath = self._zinc.compile_classpath( 'compile_classpath', ctx.target, extra_cp_entries=self._extra_compile_time_classpath) dependency_classpath = fast_relpath_collection( dependency_classpath) classpath_rel = fast_relpath_collection(classpath_abs) cp_entries = [] cp_entries.extend(classpath_rel) counter_val = str(counter()).rjust(counter.format_length(), b' ') counter_str = '[{}/{}] '.format(counter_val, counter.size) self.context.log.info( counter_str, 'Metacp-ing ', items_to_report_element(cp_entries, 'jar'), ' in ', items_to_report_element( [t.address.reference() for t in vts.targets], 'target'), ' (', ctx.target.address.spec, ').') ctx.ensure_output_dirs_exist() tgt, = vts.targets with Timer() as timer: # Step 1: Convert classpath to SemanticDB # --------------------------------------- scalac_classpath_path_entries_abs = self.tool_classpath( 'workaround-metacp-dependency-classpath') scalac_classpath_path_entries = fast_relpath_collection( scalac_classpath_path_entries_abs) rsc_index_dir = fast_relpath(ctx.rsc_index_dir, get_buildroot()) args = [ '--verbose', # NB: We need to add these extra dependencies in order to be able # to find symbols used by the scalac jars. '--dependency-classpath', os.pathsep.join( dependency_classpath + scalac_classpath_path_entries + fast_relpath_collection(self._jvm_lib_jars_abs)), # NB: The directory to dump the semanticdb jars generated by metacp. '--out', rsc_index_dir, os.pathsep.join(cp_entries), ] # NB: If we're building a scala library jar, # also request that metacp generate the indices # for the scala synthetics. if self._is_scala_core_library(tgt): args = [ '--include-scala-library-synthetics', ] + args metacp_wu = self._runtool( 'scala.meta.cli.Metacp', 'metacp', args, distribution, tgt=tgt, input_files=tuple(dependency_classpath + scalac_classpath_path_entries + classpath_rel), output_dir=rsc_index_dir) metacp_stdout = stdout_contents(metacp_wu) metacp_result = json.loads(metacp_stdout) metai_classpath = self._collect_metai_classpath( metacp_result, classpath_rel) # Step 1.5: metai Index the semanticdbs # ------------------------------------- self._run_metai_tool(distribution, metai_classpath, rsc_index_dir, tgt) abs_output = [(conf, os.path.join(get_buildroot(), x)) for conf in self._confs for x in metai_classpath] self._metacp_jars_classpath_product.add_for_target( ctx.target, abs_output, ) self._record_target_stats(tgt, len(abs_output), len([]), timer.elapsed, False, 'metacp')
def work_for_vts(vts, ctx): progress_message = ctx.target.address.spec # Capture a compilation log if requested. log_file = ctx.log_file if self._capture_log else None # Double check the cache before beginning compilation hit_cache = check_cache(vts) if not hit_cache: # Compute the compile classpath for this target. cp_entries = [ctx.classes_dir] cp_entries.extend(ClasspathUtil.compute_classpath(ctx.dependencies(self._dep_context), classpath_products, extra_compile_time_classpath, self._confs)) upstream_analysis = dict(self._upstream_analysis(compile_contexts, cp_entries)) is_incremental = should_compile_incrementally(vts, ctx) if not is_incremental: # Purge existing analysis file in non-incremental mode. safe_delete(ctx.analysis_file) # Work around https://github.com/pantsbuild/pants/issues/3670 safe_rmtree(ctx.classes_dir) tgt, = vts.targets fatal_warnings = self._compute_language_property(tgt, lambda x: x.fatal_warnings) zinc_file_manager = self._compute_language_property(tgt, lambda x: x.zinc_file_manager) with Timer() as timer: self._compile_vts(vts, ctx.target, ctx.sources, ctx.analysis_file, upstream_analysis, cp_entries, ctx.classes_dir, log_file, ctx.zinc_args_file, progress_message, tgt.platform, fatal_warnings, zinc_file_manager, counter) self._record_target_stats(tgt, len(cp_entries), len(ctx.sources), timer.elapsed, is_incremental) self._analysis_tools.relativize(ctx.analysis_file, ctx.portable_analysis_file) # Write any additional resources for this target to the target workdir. self.write_extra_resources(ctx) # Jar the compiled output. self._create_context_jar(ctx) # Update the products with the latest classes. self._register_vts([ctx]) # Once products are registered, check for unused dependencies (if enabled). if not hit_cache and self._unused_deps_check_enabled: self._check_unused_deps(ctx)