def post_process_cached_vts(self, cached_vts): """Special post processing for global scala analysis files. Class files are retrieved directly into their final locations in the global classes dir. """ self._ensure_analysis_tmpdir() # Get all the targets whose artifacts we found in the cache. cached_targets = [] for vt in cached_vts: for target in vt.targets: cached_targets.append(target) # The current global analysis may contain old data for modified targets for # which we got cache hits. We need to strip out this old analysis, to ensure # that the new data incoming from the cache doesn't collide with it during the merge. sources_to_strip = [] if os.path.exists(self._analysis_file): for target in cached_targets: sources_to_strip.extend(self._get_previous_sources_by_target(target)) # Localize the cached analyses. analyses_to_merge = [] for target in cached_targets: analysis_file = JvmCompileStrategy._analysis_for_target(self._analysis_tmpdir, target) portable_analysis_file = JvmCompileStrategy._portable_analysis_for_target(self._analysis_tmpdir, target) if os.path.exists(portable_analysis_file): self._analysis_tools.localize(portable_analysis_file, analysis_file) if os.path.exists(analysis_file): analyses_to_merge.append(analysis_file) # Merge them into the global analysis. if analyses_to_merge: with temporary_dir() as tmpdir: if sources_to_strip: throwaway = os.path.join(tmpdir, 'throwaway') trimmed_analysis = os.path.join(tmpdir, 'trimmed') self._analysis_tools.split_to_paths(self._analysis_file, [(sources_to_strip, throwaway)], trimmed_analysis) else: trimmed_analysis = self._analysis_file if os.path.exists(trimmed_analysis): analyses_to_merge.append(trimmed_analysis) tmp_analysis = os.path.join(tmpdir, 'analysis') with self.context.new_workunit(name='merge_analysis'): self._analysis_tools.merge_from_paths(analyses_to_merge, tmp_analysis) sources_by_cached_target = self._sources_for_targets(cached_targets) # Record the cached target -> sources mapping for future use. for target, sources in sources_by_cached_target.items(): self._record_previous_sources_by_target(target, sources) # Everything's good so move the merged analysis to its final location. if os.path.exists(tmp_analysis): self.move(tmp_analysis, self._analysis_file)
def post_process_cached_vts(self, cached_vts): """Special post processing for global scala analysis files. Class files are retrieved directly into their final locations in the global classes dir. """ # Get all the targets whose artifacts we found in the cache. cached_targets = [] for vt in cached_vts: for target in vt.targets: cached_targets.append(target) # The current global analysis may contain old data for modified targets for # which we got cache hits. We need to strip out this old analysis, to ensure # that the new data incoming from the cache doesn't collide with it during the merge. sources_to_strip = [] if os.path.exists(self._analysis_file): for target in cached_targets: sources_to_strip.extend(self._get_previous_sources_by_target(target)) # Localize the cached analyses. analyses_to_merge = [] for target in cached_targets: analysis_file = JvmCompileStrategy._analysis_for_target(self.analysis_tmpdir, target) portable_analysis_file = JvmCompileStrategy._portable_analysis_for_target( self.analysis_tmpdir, target) if os.path.exists(portable_analysis_file): self._analysis_tools.localize(portable_analysis_file, analysis_file) if os.path.exists(analysis_file): analyses_to_merge.append(analysis_file) # Merge them into the global analysis. if analyses_to_merge: with temporary_dir() as tmpdir: if sources_to_strip: throwaway = os.path.join(tmpdir, 'throwaway') trimmed_analysis = os.path.join(tmpdir, 'trimmed') self._analysis_tools.split_to_paths(self._analysis_file, [(sources_to_strip, throwaway)], trimmed_analysis) else: trimmed_analysis = self._analysis_file if os.path.exists(trimmed_analysis): analyses_to_merge.append(trimmed_analysis) tmp_analysis = os.path.join(tmpdir, 'analysis') with self.context.new_workunit(name='merge_analysis'): self._analysis_tools.merge_from_paths(analyses_to_merge, tmp_analysis) sources_by_cached_target = self._sources_for_targets(cached_targets) # Record the cached target -> sources mapping for future use. for target, sources in sources_by_cached_target.items(): self._record_previous_sources_by_target(target, sources) # Everything's good so move the merged analysis to its final location. if os.path.exists(tmp_analysis): self.move(tmp_analysis, self._analysis_file)
def _write_to_artifact_cache(self, analysis_file, vts, get_update_artifact_cache_work): vt_by_target = dict([(vt.target, vt) for vt in vts.versioned_targets]) vts_targets = [t for t in vts.targets if not t.has_label('no_cache')] # Determine locations for analysis files that will be split in the background. split_analysis_files = [ JvmCompileStrategy._analysis_for_target(self.analysis_tmpdir, t) for t in vts_targets] portable_split_analysis_files = [ JvmCompileStrategy._portable_analysis_for_target(self.analysis_tmpdir, t) for t in vts_targets] # Set up args for splitting the analysis into per-target files. splits = zip([self._sources_for_target(t) for t in vts_targets], split_analysis_files) splits_args_tuples = [(analysis_file, splits)] # Set up args for rebasing the splits. relativize_args_tuples = zip(split_analysis_files, portable_split_analysis_files) # Compute the classes and resources for each vts. compile_contexts = [self.compile_context(t) for t in vts_targets] vts_artifactfiles_pairs = [] classes_by_source_by_context = self.compute_classes_by_source(compile_contexts) resources_by_target = self.context.products.get_data('resources_by_target') for compile_context in compile_contexts: target = compile_context.target if target.has_label('no_cache'): continue artifacts = [] if resources_by_target is not None: for _, paths in resources_by_target[target].abs_paths(): artifacts.extend(paths) classes_by_source = classes_by_source_by_context[compile_context] for source in compile_context.sources: classes = classes_by_source.get(source, []) artifacts.extend(classes) vt = vt_by_target.get(target) if vt is not None: # NOTE: analysis_file doesn't exist yet. vts_artifactfiles_pairs.append( (vt, artifacts + [JvmCompileStrategy._portable_analysis_for_target( self.analysis_tmpdir, target)])) update_artifact_cache_work = get_update_artifact_cache_work(vts_artifactfiles_pairs) if update_artifact_cache_work: work_chain = [ Work(self._analysis_tools.split_to_paths, splits_args_tuples, 'split'), Work(self._analysis_tools.relativize, relativize_args_tuples, 'relativize'), update_artifact_cache_work ] self.context.submit_background_work_chain(work_chain, parent_workunit_name='cache')
def work(): progress_message = compile_context.target.address.spec cp_entries = self._compute_classpath_entries( compile_classpaths, target_closure, compile_context, extra_compile_time_classpath) upstream_analysis = dict( self._upstream_analysis(compile_contexts, cp_entries)) # Capture a compilation log if requested. log_file = self._capture_log_file(compile_context.target) # Mutate analysis within a temporary directory, and move it to the final location # on success. tmpdir = os.path.join(self.analysis_tmpdir, compile_context.target.id) safe_mkdir(tmpdir) tmp_analysis_file = JvmCompileStrategy._analysis_for_target( tmpdir, compile_context.target) if os.path.exists(compile_context.analysis_file): shutil.copy(compile_context.analysis_file, tmp_analysis_file) compile_vts(vts, compile_context.sources, tmp_analysis_file, upstream_analysis, cp_entries, compile_context.classes_dir, log_file, progress_message) atomic_copy(tmp_analysis_file, compile_context.analysis_file) # Update the products with the latest classes. register_vts([compile_context]) # Kick off the background artifact cache write. if update_artifact_cache_vts_work: self._write_to_artifact_cache( vts, compile_context, update_artifact_cache_vts_work)
def compile_context(self, target): analysis_file = JvmCompileStrategy._analysis_for_target(self._analysis_dir, target) classes_dir = os.path.join(self._classes_dir, target.id) return self.CompileContext(target, analysis_file, classes_dir, self._sources_for_target(target))
def work(): progress_message = vts.targets[0].address.spec cp_entries = self._compute_classpath_entries(compile_classpaths, target_closure, compile_context, extra_compile_time_classpath) upstream_analysis = dict(self._upstream_analysis(compile_contexts, cp_entries)) tmpdir = os.path.join(self.analysis_tmpdir, vts.targets[0].id) safe_mkdir(tmpdir) tmp_analysis_file = JvmCompileStrategy._analysis_for_target( tmpdir, compile_context.target) if os.path.exists(compile_context.analysis_file): shutil.copy(compile_context.analysis_file, tmp_analysis_file) compile_vts(vts, compile_context.sources, tmp_analysis_file, upstream_analysis, cp_entries, compile_context.classes_dir, progress_message) atomic_copy(tmp_analysis_file, compile_context.analysis_file) # Update the products with the latest classes. register_vts([compile_context]) # Kick off the background artifact cache write. if update_artifact_cache_vts_work: self._write_to_artifact_cache(vts, compile_context, update_artifact_cache_vts_work)
def _write_to_artifact_cache(self, vts, compile_context, get_update_artifact_cache_work): assert len(vts.targets) == 1 assert vts.targets[0] == compile_context.target # Noop if the target is uncacheable. if (compile_context.target.has_label('no_cache')): return vt = vts.versioned_targets[0] # Set up args to relativize analysis in the background. portable_analysis_file = JvmCompileStrategy._portable_analysis_for_target( self._analysis_dir, compile_context.target) relativize_args_tuple = (compile_context.analysis_file, portable_analysis_file) # Compute the classes and resources for this target. artifacts = [] resources_by_target = self.context.products.get_data('resources_by_target') if resources_by_target is not None: for _, paths in resources_by_target[compile_context.target].abs_paths(): artifacts.extend(paths) for dirpath, _, filenames in safe_walk(compile_context.classes_dir): artifacts.extend([os.path.join(dirpath, f) for f in filenames]) # Get the 'work' that will publish these artifacts to the cache. # NB: the portable analysis_file won't exist until we finish. vts_artifactfiles_pair = (vt, artifacts + [portable_analysis_file]) update_artifact_cache_work = get_update_artifact_cache_work([vts_artifactfiles_pair]) # And execute it. if update_artifact_cache_work: work_chain = [ Work(self._analysis_tools.relativize, [relativize_args_tuple], 'relativize'), update_artifact_cache_work ] self.context.submit_background_work_chain(work_chain, parent_workunit_name='cache')
def compile_context(self, target): analysis_file = JvmCompileStrategy._analysis_for_target( self._analysis_dir, target) classes_dir = os.path.join(self._classes_dir, target.id) # TODO: this will be a fairly long path. should use an id hash to shorten it if possible jar_file = os.path.join(self._jars_dir, target.id + '.jar') return IsolatedCompileContext(target, analysis_file, classes_dir, jar_file, self._sources_for_target(target))
def compile_context(self, target): analysis_file = JvmCompileStrategy._analysis_for_target(self._analysis_dir, target) classes_dir = os.path.join(self._classes_dir, target.id) # TODO: this will be a fairly long path. should use an id hash to shorten it if possible jar_file = os.path.join(self._jars_dir, target.id + '.jar') return IsolatedCompileContext(target, analysis_file, classes_dir, jar_file, self._sources_for_target(target))
def compile_context(self, target): analysis_file = JvmCompileStrategy._analysis_for_target( self._analysis_dir, target) classes_dir = os.path.join(self._classes_dir, target.id) # Generate a short unique path for the jar to allow for shorter classpaths. # TODO: likely unnecessary after https://github.com/pantsbuild/pants/issues/1988 jar_file = os.path.join( self._jars_dir, '{}.jar'.format(sha1(target.id).hexdigest()[:12])) return IsolatedCompileContext(target, analysis_file, classes_dir, jar_file, self._sources_for_target(target))
def compile_context(self, target): analysis_file = JvmCompileStrategy._analysis_for_target(self._analysis_dir, target) classes_dir = os.path.join(self._classes_dir, target.id) # Generate a short unique path for the jar to allow for shorter classpaths. # TODO: likely unnecessary after https://github.com/pantsbuild/pants/issues/1988 jar_file = os.path.join(self._jars_dir, '{}.jar'.format(sha1(target.id).hexdigest()[:12])) return IsolatedCompileContext(target, analysis_file, classes_dir, jar_file, self._sources_for_target(target))
def _write_to_artifact_cache(self, vts, compile_context, get_update_artifact_cache_work): assert len(vts.targets) == 1 assert vts.targets[0] == compile_context.target # Noop if the target is uncacheable. if (compile_context.target.has_label('no_cache')): return vt = vts.versioned_targets[0] # Set up args to relativize analysis in the background. portable_analysis_file = JvmCompileStrategy._portable_analysis_for_target( self._analysis_dir, compile_context.target) relativize_args_tuple = (compile_context.analysis_file, portable_analysis_file) # Collect the artifacts for this target. artifacts = [] def add_abs_products(p): if p: for _, paths in p.abs_paths(): artifacts.extend(paths) # Resources. resources_by_target = self.context.products.get_data( 'resources_by_target') add_abs_products(resources_by_target.get(compile_context.target)) # Classes. classes_by_target = self.context.products.get_data('classes_by_target') add_abs_products(classes_by_target.get(compile_context.target)) # Log file. log_file = self._capture_log_file(compile_context.target) if log_file and os.path.exists(log_file): artifacts.append(log_file) # Jar. if self._jar: artifacts.append(compile_context.jar_file) # Get the 'work' that will publish these artifacts to the cache. # NB: the portable analysis_file won't exist until we finish. vts_artifactfiles_pair = (vt, artifacts + [portable_analysis_file]) update_artifact_cache_work = get_update_artifact_cache_work( [vts_artifactfiles_pair]) # And execute it. if update_artifact_cache_work: work_chain = [ Work(self._analysis_tools.relativize, [relativize_args_tuple], 'relativize'), update_artifact_cache_work ] self.context.submit_background_work_chain( work_chain, parent_workunit_name='cache')
def post_process_cached_vts(self, cached_vts): """Localizes the fetched analysis for targets we found in the cache. This is the complement of `_write_to_artifact_cache`. """ compile_contexts = [] for vt in cached_vts: for target in vt.targets: compile_contexts.append(self.compile_context(target)) for compile_context in compile_contexts: portable_analysis_file = JvmCompileStrategy._portable_analysis_for_target( self._analysis_dir, compile_context.target) if os.path.exists(portable_analysis_file): self._analysis_tools.localize(portable_analysis_file, compile_context.analysis_file)
def _write_to_artifact_cache(self, vts, compile_context, get_update_artifact_cache_work): assert len(vts.targets) == 1 assert vts.targets[0] == compile_context.target # Noop if the target is uncacheable. if (compile_context.target.has_label('no_cache')): return vt = vts.versioned_targets[0] # Set up args to relativize analysis in the background. portable_analysis_file = JvmCompileStrategy._portable_analysis_for_target( self._analysis_dir, compile_context.target) relativize_args_tuple = (compile_context.analysis_file, portable_analysis_file) # Collect the artifacts for this target. artifacts = [] def add_abs_products(p): if p: for _, paths in p.abs_paths(): artifacts.extend(paths) # Resources. resources_by_target = self.context.products.get_data('resources_by_target') add_abs_products(resources_by_target.get(compile_context.target)) # Classes. classes_by_target = self.context.products.get_data('classes_by_target') add_abs_products(classes_by_target.get(compile_context.target)) # Log file. log_file = self._capture_log_file(compile_context.target) if log_file and os.path.exists(log_file): artifacts.append(log_file) # Jar. if self._jar: artifacts.append(compile_context.jar_file) # Get the 'work' that will publish these artifacts to the cache. # NB: the portable analysis_file won't exist until we finish. vts_artifactfiles_pair = (vt, artifacts + [portable_analysis_file]) update_artifact_cache_work = get_update_artifact_cache_work([vts_artifactfiles_pair]) # And execute it. if update_artifact_cache_work: work_chain = [ Work(self._analysis_tools.relativize, [relativize_args_tuple], 'relativize'), update_artifact_cache_work ] self.context.submit_background_work_chain(work_chain, parent_workunit_name='cache')
def work_for_vts(vts, compile_context, target_closure): progress_message = compile_context.target.address.spec cp_entries = self._compute_classpath_entries(compile_classpaths, target_closure, compile_context, extra_compile_time_classpath) upstream_analysis = dict(self._upstream_analysis(compile_contexts, cp_entries)) # Capture a compilation log if requested. log_file = self._capture_log_file(compile_context.target) # Double check the cache before beginning compilation if not check_cache(vts): # Mutate analysis within a temporary directory, and move it to the final location # on success. tmpdir = os.path.join(self.analysis_tmpdir, compile_context.target.id) safe_mkdir(tmpdir) tmp_analysis_file = JvmCompileStrategy._analysis_for_target( tmpdir, compile_context.target) if os.path.exists(compile_context.analysis_file): shutil.copy(compile_context.analysis_file, tmp_analysis_file) target, = vts.targets compile_vts(vts, compile_context.sources, tmp_analysis_file, upstream_analysis, cp_entries, compile_context.classes_dir, log_file, progress_message, target.platform) atomic_copy(tmp_analysis_file, compile_context.analysis_file) # Jar the compiled output. self._create_context_jar(compile_context) # Update the products with the latest classes. register_vts([compile_context]) # Kick off the background artifact cache write. if update_artifact_cache_vts_work: self._write_to_artifact_cache(vts, compile_context, update_artifact_cache_vts_work)
def _write_to_artifact_cache(self, vts, compile_context, get_update_artifact_cache_work): assert len(vts.targets) == 1 assert vts.targets[0] == compile_context.target # Noop if the target is uncacheable. if compile_context.target.has_label("no_cache"): return vt = vts.versioned_targets[0] # Set up args to relativize analysis in the background. portable_analysis_file = JvmCompileStrategy._portable_analysis_for_target( self._analysis_dir, compile_context.target ) relativize_args_tuple = (compile_context.analysis_file, portable_analysis_file) # Collect the artifacts for this target. artifacts = [] resources_by_target = self.context.products.get_data("resources_by_target") if resources_by_target is not None: for _, paths in resources_by_target[compile_context.target].abs_paths(): artifacts.extend(paths) target_classes = self.context.products.get_data("classes_by_target").get(compile_context.target) for _, classfiles in target_classes.abs_paths(): artifacts.extend(classfiles) log_file = self._capture_log_file(compile_context.target) if log_file and os.path.exists(log_file): artifacts.append(log_file) # Get the 'work' that will publish these artifacts to the cache. # NB: the portable analysis_file won't exist until we finish. vts_artifactfiles_pair = (vt, artifacts + [portable_analysis_file]) update_artifact_cache_work = get_update_artifact_cache_work([vts_artifactfiles_pair]) # And execute it. if update_artifact_cache_work: work_chain = [ Work(self._analysis_tools.relativize, [relativize_args_tuple], "relativize"), update_artifact_cache_work, ] self.context.submit_background_work_chain(work_chain, parent_workunit_name="cache")
def compile_context(self, target): analysis_file = JvmCompileStrategy._analysis_for_target( self._analysis_dir, target) classes_dir = os.path.join(self._classes_dir, target.id) return self.CompileContext(target, analysis_file, classes_dir, self._sources_for_target(target))