def work(): progress_message = vts.targets[0].address.spec cp_entries = self._compute_classpath_entries(compile_classpaths, target_closure, compile_context, extra_compile_time_classpath) upstream_analysis = dict(self._upstream_analysis(compile_contexts, cp_entries)) tmpdir = os.path.join(self.analysis_tmpdir, vts.targets[0].id) safe_mkdir(tmpdir) tmp_analysis_file = JvmCompileStrategy._analysis_for_target( tmpdir, compile_context.target) if os.path.exists(compile_context.analysis_file): shutil.copy(compile_context.analysis_file, tmp_analysis_file) compile_vts(vts, compile_context.sources, tmp_analysis_file, upstream_analysis, cp_entries, compile_context.classes_dir, progress_message) atomic_copy(tmp_analysis_file, compile_context.analysis_file) # Update the products with the latest classes. register_vts([compile_context]) # Kick off the background artifact cache write. if update_artifact_cache_vts_work: self._write_to_artifact_cache(vts, compile_context, update_artifact_cache_vts_work)
def work(): progress_message = compile_context.target.address.spec cp_entries = self._compute_classpath_entries( compile_classpaths, target_closure, compile_context, extra_compile_time_classpath) upstream_analysis = dict( self._upstream_analysis(compile_contexts, cp_entries)) # Capture a compilation log if requested. log_file = self._capture_log_file(compile_context.target) # Mutate analysis within a temporary directory, and move it to the final location # on success. tmpdir = os.path.join(self.analysis_tmpdir, compile_context.target.id) safe_mkdir(tmpdir) tmp_analysis_file = JvmCompileStrategy._analysis_for_target( tmpdir, compile_context.target) if os.path.exists(compile_context.analysis_file): shutil.copy(compile_context.analysis_file, tmp_analysis_file) compile_vts(vts, compile_context.sources, tmp_analysis_file, upstream_analysis, cp_entries, compile_context.classes_dir, log_file, progress_message) atomic_copy(tmp_analysis_file, compile_context.analysis_file) # Update the products with the latest classes. register_vts([compile_context]) # Kick off the background artifact cache write. if update_artifact_cache_vts_work: self._write_to_artifact_cache( vts, compile_context, update_artifact_cache_vts_work)
def execute(self): targets = self.get_targets(self._is_python_lambda) with self.invalidated( targets=targets, invalidate_dependents=True) as invalidation_check: python_lambda_product = self.context.products.get_data( "python_aws_lambda", dict) for vt in invalidation_check.all_vts: lambda_path = os.path.join(vt.results_dir, "{}.pex".format(vt.target.name)) if not vt.valid: self.context.log.debug( "Existing lambda for {} is invalid, rebuilding".format( vt.target)) self._create_lambda(vt.target, lambda_path) else: self.context.log.debug( "Using existing lambda for {}".format(vt.target)) python_lambda_product[vt.target] = lambda_path self.context.log.debug("created {}".format( os.path.relpath(lambda_path, get_buildroot()))) # Put a copy in distdir. lambda_copy = os.path.join(self.get_options().pants_distdir, os.path.basename(lambda_path)) safe_mkdir_for(lambda_copy) atomic_copy(lambda_path, lambda_copy) self.context.log.info("created lambda {}".format( os.path.relpath(lambda_copy, get_buildroot())))
def shade_jar(self, shading_rules, jar_path): """Shades a jar using the shading rules from the given jvm_binary. This *overwrites* the existing jar file at ``jar_path``. :param shading_rules: predefined rules for shading :param jar_path: The filepath to the jar that should be shaded. """ self.context.log.debug('Shading {}.'.format(jar_path)) with temporary_dir() as tempdir: output_jar = os.path.join(tempdir, os.path.basename(jar_path)) with self.shader.binary_shader_for_rules( output_jar, jar_path, shading_rules) as shade_runner: result = execute_runner( shade_runner, workunit_factory=self.context.new_workunit, workunit_name='jarjar') if result != 0: raise TaskError( 'Shading tool failed to shade {0} (error code {1})'. format(jar_path, result)) if not os.path.exists(output_jar): raise TaskError( 'Shading tool returned success for {0}, but ' 'the output jar was not found at {1}'.format( jar_path, output_jar)) atomic_copy(output_jar, jar_path) return jar_path
def execute(self): binaries = self.context.targets(self.is_binary) # Check for duplicate binary names, since we write the pexes to <dist>/<name>.pex. names = {} for binary in binaries: name = binary.name if name in names: raise TaskError('Cannot build two binaries with the same name in a single invocation. ' '{} and {} both have the name {}.'.format(binary, names[name], name)) names[name] = binary with self.invalidated(binaries, invalidate_dependents=True) as invalidation_check: python_deployable_archive = self.context.products.get('deployable_archives') python_pex_product = self.context.products.get('pex_archives') for vt in invalidation_check.all_vts: pex_path = os.path.join(vt.results_dir, '{}.pex'.format(vt.target.name)) if not vt.valid: self.context.log.debug('cache for {} is invalid, rebuilding'.format(vt.target)) self._create_binary(vt.target, vt.results_dir) else: self.context.log.debug('using cache for {}'.format(vt.target)) basename = os.path.basename(pex_path) python_pex_product.add(vt.target, os.path.dirname(pex_path)).append(basename) python_deployable_archive.add(vt.target, os.path.dirname(pex_path)).append(basename) self.context.log.debug('created {}'.format(os.path.relpath(pex_path, get_buildroot()))) # Create a copy for pex. pex_copy = os.path.join(self._distdir, os.path.basename(pex_path)) safe_mkdir_for(pex_copy) atomic_copy(pex_path, pex_copy) self.context.log.info('created pex {}'.format(os.path.relpath(pex_copy, get_buildroot())))
def work_for_vts(vts, compile_context, target_closure): progress_message = compile_context.target.address.spec cp_entries = self._compute_classpath_entries(classpath_products, target_closure, compile_context, extra_compile_time_classpath) upstream_analysis = dict(self._upstream_analysis(compile_contexts, cp_entries)) # Capture a compilation log if requested. log_file = self._capture_log_file(compile_context.target) # Double check the cache before beginning compilation hit_cache = check_cache(vts) incremental = False if not hit_cache: # Mutate analysis within a temporary directory, and move it to the final location # on success. tmpdir = os.path.join(self.analysis_tmpdir, compile_context.target.id) safe_mkdir(tmpdir) tmp_analysis_file = self._analysis_for_target( tmpdir, compile_context.target) # If the analysis exists for this context, it is an incremental compile. if os.path.exists(compile_context.analysis_file): incremental = True shutil.copy(compile_context.analysis_file, tmp_analysis_file) target, = vts.targets compile_vts(vts, compile_context.sources, tmp_analysis_file, upstream_analysis, cp_entries, compile_context.classes_dir, log_file, progress_message, target.platform) atomic_copy(tmp_analysis_file, compile_context.analysis_file) # Jar the compiled output. self._create_context_jar(compile_context) # Update the products with the latest classes. register_vts([compile_context]) # We write to the cache only if we didn't hit during the double check, and optionally # only for clean builds. is_cacheable = not hit_cache and (self.get_options().incremental_caching or not incremental) self.context.log.debug( 'Completed compile for {}. ' 'Hit cache: {}, was incremental: {}, is cacheable: {}, cache writes enabled: {}.'.format( compile_context.target.address.spec, hit_cache, incremental, is_cacheable, update_artifact_cache_vts_work is not None )) if is_cacheable and update_artifact_cache_vts_work: # Kick off the background artifact cache write. self._write_to_artifact_cache(vts, compile_context, update_artifact_cache_vts_work)
def execute(self): binaries = self.context.targets(self.is_binary) # Check for duplicate binary names, since we write the pexes to <dist>/<name>.pex. names = {} for binary in binaries: name = binary.name if name in names: raise TaskError('Cannot build two binaries with the same name in a single invocation. ' '{} and {} both have the name {}.'.format(binary, names[name], name)) names[name] = binary with self.invalidated(binaries, invalidate_dependents=True) as invalidation_check: python_deployable_archive = self.context.products.get('deployable_archives') python_pex_product = self.context.products.get('pex_archives') for vt in invalidation_check.all_vts: pex_path = os.path.join(vt.results_dir, '{}.pex'.format(vt.target.name)) if not vt.valid: self.context.log.debug('cache for {} is invalid, rebuilding'.format(vt.target)) self._create_binary(vt.target, vt.results_dir) else: self.context.log.debug('using cache for {}'.format(vt.target)) basename = os.path.basename(pex_path) python_pex_product.add(binary, os.path.dirname(pex_path)).append(basename) python_deployable_archive.add(binary, os.path.dirname(pex_path)).append(basename) self.context.log.debug('created {}'.format(os.path.relpath(pex_path, get_buildroot()))) # Create a copy for pex. pex_copy = os.path.join(self._distdir, os.path.basename(pex_path)) safe_mkdir_for(pex_copy) atomic_copy(pex_path, pex_copy) self.context.log.info('created pex {}'.format(os.path.relpath(pex_copy, get_buildroot())))
def test_atomic_copy(self): with temporary_file() as src: src.write(src.name) src.flush() with temporary_file() as dst: atomic_copy(src.name, dst.name) dst.close() with open(dst.name) as new_dst: self.assertEquals(src.name, new_dst.read())
def work_for_vts(vts, compile_context, target_closure): progress_message = compile_context.target.address.spec cp_entries = self._compute_classpath_entries( classpath_products, target_closure, compile_context, extra_compile_time_classpath) upstream_analysis = dict( self._upstream_analysis(compile_contexts, cp_entries)) # Capture a compilation log if requested. log_file = self._capture_log_file(compile_context.target) # Double check the cache before beginning compilation hit_cache = check_cache(vts) incremental = False if not hit_cache: # Mutate analysis within a temporary directory, and move it to the final location # on success. tmpdir = os.path.join(self.analysis_tmpdir, compile_context.target.id) safe_mkdir(tmpdir) tmp_analysis_file = self._analysis_for_target( tmpdir, compile_context.target) # If the analysis exists for this context, it is an incremental compile. if os.path.exists(compile_context.analysis_file): incremental = True shutil.copy(compile_context.analysis_file, tmp_analysis_file) target, = vts.targets compile_vts(vts, compile_context.sources, tmp_analysis_file, upstream_analysis, cp_entries, compile_context.classes_dir, log_file, progress_message, target.platform) atomic_copy(tmp_analysis_file, compile_context.analysis_file) # Jar the compiled output. self._create_context_jar(compile_context) # Update the products with the latest classes. register_vts([compile_context]) # We write to the cache only if we didn't hit during the double check, and optionally # only for clean builds. is_cacheable = not hit_cache and ( self.get_options().incremental_caching or not incremental) self.context.log.debug( 'Completed compile for {}. ' 'Hit cache: {}, was incremental: {}, is cacheable: {}, cache writes enabled: {}.' .format(compile_context.target.address.spec, hit_cache, incremental, is_cacheable, update_artifact_cache_vts_work is not None)) if is_cacheable and update_artifact_cache_vts_work: # Kick off the background artifact cache write. self._write_to_artifact_cache(vts, compile_context, update_artifact_cache_vts_work)
def test_atomic_copy(self): with temporary_file() as src: src.write(src.name.encode('utf-8')) src.flush() with temporary_file() as dst: atomic_copy(src.name, dst.name) dst.close() with open(dst.name) as new_dst: self.assertEquals(src.name, new_dst.read()) self.assertEqual(os.stat(src.name).st_mode, os.stat(dst.name).st_mode)
def test_atomic_copy(self) -> None: with temporary_file() as src: src.write(src.name.encode()) src.flush() with temporary_file() as dst: atomic_copy(src.name, dst.name) dst.close() with open(dst.name, "r") as new_dst: self.assertEqual(src.name, new_dst.read()) self.assertEqual(os.stat(src.name).st_mode, os.stat(dst.name).st_mode)
def _copy_ivy_reports(cls, workdir_report_paths_by_conf, confs, ivy_cache_dir, resolve_hash_name): for conf in confs: ivy_cache_report_path = IvyUtils.xml_report_path(ivy_cache_dir, resolve_hash_name, conf) workdir_report_path = workdir_report_paths_by_conf[conf] try: atomic_copy(ivy_cache_report_path, workdir_report_path) except IOError as e: raise cls.IvyError('Failed to copy report into workdir from {} to {}: {}' .format(ivy_cache_report_path, workdir_report_path, e))
def publish_results(self, dist_dir, use_basename_prefix, vt, bundle_dir, archivepath, id, archive_ext): """Publish a copy of the bundle and archive from the results dir in dist.""" # TODO (from mateor) move distdir management somewhere more general purpose. name = vt.target.basename if use_basename_prefix else id bundle_copy = os.path.join(dist_dir, '{}-bundle'.format(name)) absolute_symlink(bundle_dir, bundle_copy) self.context.log.info( 'created bundle copy {}'.format(os.path.relpath(bundle_copy, get_buildroot()))) if archivepath: ext = archive.archive_extensions.get(archive_ext, archive_ext) archive_copy = os.path.join(dist_dir,'{}.{}'.format(name, ext)) safe_mkdir_for(archive_copy) # Ensure parent dir exists atomic_copy(archivepath, archive_copy) self.context.log.info( 'created archive copy {}'.format(os.path.relpath(archive_copy, get_buildroot())))
def work_for_vts(vts, compile_context, target_closure): progress_message = compile_context.target.address.spec cp_entries = self._compute_classpath_entries(compile_classpaths, target_closure, compile_context, extra_compile_time_classpath) upstream_analysis = dict(self._upstream_analysis(compile_contexts, cp_entries)) # Capture a compilation log if requested. log_file = self._capture_log_file(compile_context.target) # Double check the cache before beginning compilation if not check_cache(vts): # Mutate analysis within a temporary directory, and move it to the final location # on success. tmpdir = os.path.join(self.analysis_tmpdir, compile_context.target.id) safe_mkdir(tmpdir) tmp_analysis_file = self._analysis_for_target( tmpdir, compile_context.target) if os.path.exists(compile_context.analysis_file): shutil.copy(compile_context.analysis_file, tmp_analysis_file) target, = vts.targets compile_vts(vts, compile_context.sources, tmp_analysis_file, upstream_analysis, cp_entries, compile_context.classes_dir, log_file, progress_message, target.platform) atomic_copy(tmp_analysis_file, compile_context.analysis_file) # Jar the compiled output. self._create_context_jar(compile_context) # Update the products with the latest classes. register_vts([compile_context]) # Kick off the background artifact cache write. if update_artifact_cache_vts_work: self._write_to_artifact_cache(vts, compile_context, update_artifact_cache_vts_work)
def execute(self): dist_targets = self.context.targets(is_local_python_dist) local_wheels_product = self.context.products.get('local_wheels') if not local_wheels_product: return safe_mkdir(self.dist_dir) # Make sure dist dir is present. for t in dist_targets: # Copy generated wheel files to dist folder target_local_wheels = local_wheels_product.get(t) if not target_local_wheels: continue for output_dir, base_names in target_local_wheels.items(): for base_name in base_names: wheel_output = os.path.join(output_dir, base_name) self.context.log.debug('found local built wheels {}'.format(wheel_output)) # Create a copy for wheel in dist dir. wheel_copy = os.path.join(self.dist_dir, base_name) atomic_copy(wheel_output, wheel_copy) self.context.log.info( 'created wheel {}'.format(os.path.relpath(wheel_copy, get_buildroot())))
def execute(self): targets = self.get_targets(self._is_python_lambda) with self.invalidated(targets=targets) as invalidation_check: python_lambda_product = self.context.products.get_data('python_aws_lambda', dict) for vt in invalidation_check.all_vts: lambda_path = os.path.join(vt.results_dir, '{}.pex'.format(vt.target.name)) if not vt.valid: self.context.log.debug('Existing lambda for {} is invalid, rebuilding'.format(vt.target)) self._create_lambda(vt.target, lambda_path) else: self.context.log.debug('Using existing lambda for {}'.format(vt.target)) python_lambda_product[vt.target] = lambda_path self.context.log.debug('created {}'.format(os.path.relpath(lambda_path, get_buildroot()))) # Put a copy in distdir. lambda_copy = os.path.join(self.get_options().pants_distdir, os.path.basename(lambda_path)) safe_mkdir_for(lambda_copy) atomic_copy(lambda_path, lambda_copy) self.context.log.info('created lambda {}'.format( os.path.relpath(lambda_copy, get_buildroot())))
def execute(self): dist_targets = self.context.targets(is_local_python_dist) local_wheels_product = self.context.products.get('local_wheels') if not local_wheels_product: return safe_mkdir(self.dist_dir) # Make sure dist dir is present. for t in dist_targets: # Copy generated wheel files to dist folder target_local_wheels = local_wheels_product.get(t) if not target_local_wheels: continue for output_dir, base_names in target_local_wheels.items(): for base_name in base_names: wheel_output = os.path.join(output_dir, base_name) self.context.log.debug(f'found local built wheels {wheel_output}') # Create a copy for wheel in dist dir. wheel_copy = os.path.join(self.dist_dir, base_name) atomic_copy(wheel_output, wheel_copy) self.context.log.info( 'created wheel {}'.format(os.path.relpath(wheel_copy, get_buildroot())))
def shade_jar(self, shading_rules, jar_path): """Shades a jar using the shading rules from the given jvm_binary. This *overwrites* the existing jar file at ``jar_path``. :param shading_rules: predefined rules for shading :param jar_path: The filepath to the jar that should be shaded. """ self.context.log.debug('Shading {}.'.format(jar_path)) with temporary_dir() as tempdir: output_jar = os.path.join(tempdir, os.path.basename(jar_path)) with self.shader.binary_shader_for_rules(output_jar, jar_path, shading_rules) as shade_runner: result = execute_runner(shade_runner, workunit_factory=self.context.new_workunit, workunit_name='jarjar') if result != 0: raise TaskError('Shading tool failed to shade {0} (error code {1})'.format(jar_path, result)) if not os.path.exists(output_jar): raise TaskError('Shading tool returned success for {0}, but ' 'the output jar was not found at {1}'.format(jar_path, output_jar)) atomic_copy(output_jar, jar_path) return jar_path
def _do_resolve(self, confs, executor, extra_args, global_vts, pinned_artifacts, raw_target_classpath_file, resolve_hash_name, resolve_workdir, workunit_name): safe_mkdir(resolve_workdir) ivy = Bootstrapper.default_ivy(bootstrap_workunit_factory=self.context.new_workunit) with safe_concurrent_creation(raw_target_classpath_file) as raw_target_classpath_file_tmp: args = ['-cachepath', raw_target_classpath_file_tmp] + extra_args targets = global_vts.targets # TODO(John Sirois): merge the code below into IvyUtils or up here; either way, better # diagnostics can be had in `IvyUtils.generate_ivy` if this is done. # See: https://github.com/pantsbuild/pants/issues/2239 jars, global_excludes = IvyUtils.calculate_classpath(targets) # Don't pass global excludes to ivy when using soft excludes. if self.get_options().soft_excludes: global_excludes = [] ivyxml = self._ivy_xml_path(resolve_workdir) with IvyUtils.ivy_lock: try: IvyUtils.generate_ivy(targets, jars, global_excludes, ivyxml, confs, resolve_hash_name, pinned_artifacts) except IvyUtils.IvyError as e: raise self.Error('Failed to prepare ivy resolve: {}'.format(e)) self._exec_ivy(ivy, executor, confs, ivyxml, args, workunit_name) # Copy ivy resolve file into resolve workdir. for conf in confs: atomic_copy(IvyUtils.xml_report_path(self.ivy_cache_dir, resolve_hash_name, conf), self._resolve_report_path(resolve_workdir, conf)) if not os.path.exists(raw_target_classpath_file_tmp): raise self.Error('Ivy failed to create classpath file at {}' .format(raw_target_classpath_file_tmp)) logger.debug('Moved ivy classfile file to {dest}'.format(dest=raw_target_classpath_file))