def test_tar(self): self.round_trip(archiver('tar'), expected_ext='tar', empty_dirs=True) self.round_trip(archiver('tgz'), expected_ext='tar.gz', empty_dirs=True) self.round_trip(archiver('tbz2'), expected_ext='tar.bz2', empty_dirs=True)
def _extract_jar(self, jar_path): """Extracts the jar to a subfolder of workdir/extracted and returns the path to it.""" with open(jar_path, 'rb') as f: outdir = os.path.join(self.workdir, 'extracted', sha1(f.read()).hexdigest()) if not os.path.exists(outdir): _, extension = os.path.splitext(jar_path) archiver(extension[1:]).extract(jar_path, outdir) self.context.log.debug('Extracting archive at {jar_path}.'.format(jar_path=jar_path)) else: self.context.log.debug('Archive already extracted at {jar_path}.'.format(jar_path=jar_path)) return outdir
def test_zip_filter(self): def do_filter(path): return path == 'allowed.txt' with temporary_dir() as fromdir: touch(os.path.join(fromdir, 'allowed.txt')) touch(os.path.join(fromdir, 'disallowed.txt')) with temporary_dir() as archivedir: archive = archiver('zip').create(fromdir, archivedir, 'archive') with temporary_dir() as todir: archiver('zip').extract(archive, todir, filter_func=do_filter) self.assertEquals(set(['allowed.txt']), self._listtree(todir, empty_dirs=False))
def check_archive_with_flags(archive_format, dereference): with temporary_dir() as fromdir: filename = os.path.join(fromdir, 'a') linkname = os.path.join(fromdir, 'link_to_a') touch(filename) relative_symlink(filename, linkname) with temporary_dir() as archivedir: archive = archiver(archive_format).create(fromdir, archivedir, 'archive', dereference=dereference) with temporary_dir() as todir: archiver(archive_format).extract(archive, todir) extracted_linkname = os.path.join(todir, 'link_to_a') assertion = self.assertFalse if dereference else self.assertTrue assertion(os.path.islink(extracted_linkname)) assertion(os.path.samefile(extracted_linkname, os.path.join(todir, 'a')))
def execute(self): bundleable_js = self.context.products.get_data('bundleable_js') bundle_archive_product = self.context.products.get('deployable_archives') dirutil.safe_mkdir(self._outdir) # Make sure dist dir is present. for target in self.context.target_roots: if self.is_node_bundle(target): archiver = archive.archiver(target.payload.archive) for _, abs_paths in bundleable_js[target.node_module].abs_paths(): for abs_path in abs_paths: # build_dir is a symlink. Since dereference option for tar is set to False, we need to # dereference manually to archive the linked build dir. build_dir = os.path.realpath(abs_path) self.context.log.debug('archiving {}'.format(build_dir)) archivepath = archiver.create( build_dir, self._outdir, target.package_name, prefix=None, dereference=False ) bundle_archive_product.add( target, os.path.dirname(archivepath)).append(os.path.basename(archivepath)) self.context.log.info( 'created {}'.format(os.path.relpath(archivepath, get_buildroot())))
def execute(self): targets_to_bundle = self.context.targets(self.App.is_app) if self.get_options().use_basename_prefix: self.check_basename_conflicts([t for t in self.context.target_roots if t in targets_to_bundle]) with self.invalidated(targets_to_bundle, invalidate_dependents=True) as invalidation_check: jvm_bundles_product = self.context.products.get('jvm_bundles') bundle_archive_product = self.context.products.get('deployable_archives') jvm_archive_product = self.context.products.get('jvm_archives') for vt in invalidation_check.all_vts: app = self.App.create_app(vt.target, self._resolved_option(vt.target, 'deployjar'), self._resolved_option(vt.target, 'archive')) archiver = archive.archiver(app.archive) if app.archive else None bundle_dir = self._get_bundle_dir(app, vt.results_dir) ext = archive.archive_extensions.get(app.archive, app.archive) filename = '{}.{}'.format(app.id, ext) archive_path = os.path.join(vt.results_dir, filename) if app.archive else '' if not vt.valid: self.bundle(app, vt.results_dir) if app.archive: archiver.create(bundle_dir, vt.results_dir, app.id) self._add_product(jvm_bundles_product, app, bundle_dir) if archiver: self._add_product(bundle_archive_product, app, archive_path) self._add_product(jvm_archive_product, app, archive_path) # For root targets, create symlink. if vt.target in self.context.target_roots: self._store_results(vt, bundle_dir, archive_path, app)
def execute(self): targets_to_bundle = self.context.targets(PythonApp.is_python_app) with self.invalidated( targets_to_bundle, invalidate_dependents=True) as invalidation_check: bundle_archive_product = self.context.products.get( self._DEPLOYABLE_ARCHIVES) for vt in invalidation_check.all_vts: bundle_dir = self.get_bundle_dir(vt.target.id, vt.results_dir) archive_format = self.resolved_option(self.get_options(), vt.target, 'archive') archiver = archive.archiver( archive_format) if archive_format else None archive_path = self._get_archive_path(vt, archive_format) if not vt.valid: # Only recreate the bundle/archive if it's changed self._bundle(vt.target, bundle_dir) if archiver: archiver.create(bundle_dir, vt.results_dir, vt.target.id) self.context.log.info('created archive {}'.format( os.path.relpath(archive_path, get_buildroot()))) if archiver: bundle_archive_product.add( vt.target, os.path.dirname(archive_path)).append( os.path.basename(archive_path)) if vt.target in self.context.target_roots: # Always publish bundle/archive in dist self.publish_results(self.get_options().pants_distdir, False, vt, bundle_dir, archive_path, vt.target.id, archive_format)
def select(self, supportdir, version, name, platform_dependent, archive_type): """Fetches a file, unpacking it if necessary.""" if archive_type is None: return self._select_file(supportdir, version, name, platform_dependent) selected_archiver = archiver(archive_type) return self._select_archive(supportdir, version, name, platform_dependent, selected_archiver)
def _extract_archive(self, archive_path): with temporary_dir() as temp_dir: _, extension = os.path.splitext(archive_path) print (extension) if extension == '.jar': extraction_archiver = archiver('zip') else: extraction_archiver = archiver_for_path(os.path.basename(archive_path)) extraction_archiver.extract(archive_path, temp_dir) yield temp_dir
def execute(self): archiver = archive.archiver(self._archiver_type) if self._archiver_type else None for target in self.context.target_roots: for app in map(self.App, filter(self.App.is_app, [target])): basedir = self.bundle(app) if archiver: archivepath = archiver.create( basedir, self._outdir, app.basename, prefix=app.basename if self._prefix else None ) self.context.log.info("created {}".format(os.path.relpath(archivepath, get_buildroot())))
def _extract_archive(self, archive_path): with temporary_dir() as temp_dir: _, extension = os.path.splitext(archive_path) print(extension) if extension == '.jar': extraction_archiver = archiver('zip') else: extraction_archiver = archiver_for_path( os.path.basename(archive_path)) extraction_archiver.extract(archive_path, temp_dir) yield temp_dir
def execute(self): use_basename_prefix = self.get_options().use_basename_prefix if use_basename_prefix: # NB(peiyu) This special casing is confusing especially given we already fail # when duplicate basenames are detected. It's added because of the existing # user experience. Turns out a `jvm_app` that depends on another `jvm_binary` # of the same basename is fairly common. In this case, using just # `target_roots` instead of all transitive targets will reduce the chance users # see their bundle command fail due to basename conflicts. We should eventually # get rid of this special case. targets_to_bundle = self.context.target_roots else: targets_to_bundle = self.context.targets() apps = [] for target in targets_to_bundle: if self.App.is_app(target): apps.append( self.App.create_app( target, self._resolved_option(target, 'deployjar'), self._resolved_option(target, 'archive'), use_basename_prefix=use_basename_prefix)) if use_basename_prefix: self.check_basename_conflicts(apps) # NB(peiyu): performance hack to convert loose directories in classpath into jars. This is # more efficient than loading them as individual files. runtime_classpath = self.context.products.get_data('runtime_classpath') targets_to_consolidate = self.find_consolidate_classpath_candidates( runtime_classpath, self.context.targets(**self._target_closure_kwargs), ) self.consolidate_classpath(targets_to_consolidate, runtime_classpath) for app in apps: archiver = archive.archiver(app.archive) if app.archive else None basedir = self.bundle(app) # NB(Eric Ayers): Note that this product is not housed/controlled under .pants.d/ Since # the bundle is re-created every time, this shouldn't cause a problem, but if we ever # expect the product to be cached, a user running an 'rm' on the dist/ directory could # cause inconsistencies. jvm_bundles_product = self.context.products.get('jvm_bundles') jvm_bundles_product.add(app.target, os.path.dirname(basedir)).append( os.path.basename(basedir)) if archiver: archivepath = archiver.create(basedir, self.get_options().pants_distdir, app.basename) self.context.log.info('created {}'.format( os.path.relpath(archivepath, get_buildroot())))
def check_archive_with_flags(archive_format, dereference): with temporary_dir() as fromdir: filename = os.path.join(fromdir, 'a') linkname = os.path.join(fromdir, 'link_to_a') touch(filename) relative_symlink(filename, linkname) with temporary_dir() as archivedir: archive = archiver(archive_format).create( fromdir, archivedir, 'archive', dereference=dereference) with temporary_dir() as todir: archiver(archive_format).extract(archive, todir) extracted_linkname = os.path.join(todir, 'link_to_a') assertion = self.assertFalse if dereference else self.assertTrue assertion(os.path.islink(extracted_linkname)) assertion( os.path.samefile(extracted_linkname, os.path.join(todir, 'a')))
def execute(self): archiver = archive.archiver(self._archiver_type) if self._archiver_type else None for target in self.context.target_roots: for app in map(self.App, filter(self.App.is_app, [target])): basedir = self.bundle(app) if archiver: archivepath = archiver.create( basedir, self._outdir, app.basename, prefix=app.basename if self._prefix else None ) self.context.log.info('created %s' % os.path.relpath(archivepath, get_buildroot()))
def execute(self): use_basename_prefix = self.get_options().use_basename_prefix if use_basename_prefix: # NB(peiyu) This special casing is confusing especially given we already fail # when duplicate basenames are detected. It's added because of the existing # user experience. Turns out a `jvm_app` that depends on another `jvm_binary` # of the same basename is fairly common. In this case, using just # `target_roots` instead of all transitive targets will reduce the chance users # see their bundle command fail due to basename conflicts. We should eventually # get rid of this special case. targets_to_bundle = self.context.target_roots else: targets_to_bundle = self.context.targets() apps = [] for target in targets_to_bundle: if self.App.is_app(target): apps.append(self.App.create_app(target, self._resolved_option(target, 'deployjar'), self._resolved_option(target, 'archive'), use_basename_prefix=use_basename_prefix)) if use_basename_prefix: self.check_basename_conflicts(apps) # NB(peiyu): performance hack to convert loose directories in classpath into jars. This is # more efficient than loading them as individual files. runtime_classpath = self.context.products.get_data('runtime_classpath') targets_to_consolidate = self.find_consolidate_classpath_candidates( runtime_classpath, self.context.targets(**self._target_closure_kwargs), ) self.consolidate_classpath(targets_to_consolidate, runtime_classpath) for app in apps: archiver = archive.archiver(app.archive) if app.archive else None basedir = self.bundle(app) # NB(Eric Ayers): Note that this product is not housed/controlled under .pants.d/ Since # the bundle is re-created every time, this shouldn't cause a problem, but if we ever # expect the product to be cached, a user running an 'rm' on the dist/ directory could # cause inconsistencies. jvm_bundles_product = self.context.products.get('jvm_bundles') jvm_bundles_product.add(app.target, os.path.dirname(basedir)).append(os.path.basename(basedir)) if archiver: archivepath = archiver.create( basedir, self.get_options().pants_distdir, app.basename ) self.context.log.info('created {}'.format(os.path.relpath(archivepath, get_buildroot())))
def execute(self, _): archiver = archive.archiver(self.archiver_type) if self.archiver_type else None for target in self.context.target_roots: for app in map(self.App, filter(self.App.is_app, target.resolve())): basedir = self.bundle(app) if archiver: archivemap = self.context.products.get(self.archiver_type) archivepath = archiver.create( basedir, self.outdir, app.basename, prefix=app.basename if self.prefix else None ) archivemap.add(app, self.outdir, [archivepath]) self.context.log.info('created %s' % os.path.relpath(archivepath, get_buildroot()))
def execute(self): archiver = archive.archiver(self._archiver_type) if self._archiver_type else None for target in self.context.target_roots: for app in map(self.App, filter(self.App.is_app, [target])): basedir = self.bundle(app) # NB(Eric Ayers): Note that this product is not housed/controlled under .pants.d/ Since # the bundle is re-created every time, this shouldn't cause a problem, but if we ever # expect the product to be cached, a user running an 'rm' on the dist/ directory could # cause inconsistencies. jvm_bundles_product = self.context.products.get("jvm_bundles") jvm_bundles_product.add(target, os.path.dirname(basedir)).append(os.path.basename(basedir)) if archiver: archivepath = archiver.create( basedir, self._outdir, app.basename, prefix=app.basename if self._prefix else None ) self.context.log.info("created {}".format(os.path.relpath(archivepath, get_buildroot())))
def execute(self): # NB(peiyu): performance hack to convert loose directories in classpath into jars. This is # more efficient than loading them as individual files. runtime_classpath = self.context.products.get_data('runtime_classpath') # TODO (from mateor) The consolidate classpath is something that we could do earlier in the # pipeline and it would be nice to just add those unpacked classed to a product and get the # consolidated classpath for free. targets_to_consolidate = self.find_consolidate_classpath_candidates( runtime_classpath, self.context.targets(**self._target_closure_kwargs), ) self.consolidate_classpath(targets_to_consolidate, runtime_classpath) targets_to_bundle = self.context.targets(self.App.is_app) if self.get_options().use_basename_prefix: self.check_basename_conflicts([t for t in self.context.target_roots if t in targets_to_bundle]) with self.invalidated(targets_to_bundle, invalidate_dependents=True) as invalidation_check: jvm_bundles_product = self.context.products.get('jvm_bundles') bundle_archive_product = self.context.products.get('deployable_archives') jvm_archive_product = self.context.products.get('jvm_archives') for vt in invalidation_check.all_vts: app = self.App.create_app(vt.target, self._resolved_option(vt.target, 'deployjar'), self._resolved_option(vt.target, 'archive')) archiver = archive.archiver(app.archive) if app.archive else None bundle_dir = self._get_bundle_dir(app, vt.results_dir) ext = archive.archive_extensions.get(app.archive, app.archive) filename = '{}.{}'.format(app.id, ext) archive_path = os.path.join(vt.results_dir, filename) if app.archive else '' if not vt.valid: self.bundle(app, vt.results_dir) if app.archive: archiver.create(bundle_dir, vt.results_dir, app.id) self._add_product(jvm_bundles_product, app, bundle_dir) if archiver: self._add_product(bundle_archive_product, app, archive_path) self._add_product(jvm_archive_product, app, archive_path) # For root targets, create symlink. if vt.target in self.context.target_roots: self._store_results(vt, bundle_dir, archive_path, app)
def execute(self): archiver = archive.archiver(self._archiver_type) if self._archiver_type else None for target in self.context.target_roots: for app in map(self.App, filter(self.App.is_app, [target])): basedir = self.bundle(app) # NB(Eric Ayers): Note that this product is not housed/controlled under .pants.d/ Since # the bundle is re-created every time, this shouldn't cause a problem, but if we ever # expect the product to be cached, a user running an 'rm' on the dist/ directory could # cause inconsistencies. jvm_bundles_product = self.context.products.get('jvm_bundles') jvm_bundles_product.add(target, os.path.dirname(basedir)).append(os.path.basename(basedir)) if archiver: archivepath = archiver.create( basedir, self._outdir, app.basename, prefix=app.basename if self._prefix else None ) self.context.log.info('created {}'.format(os.path.relpath(archivepath, get_buildroot())))
def test_zip(self): self.round_trip(archiver('zip'), expected_ext='zip', empty_dirs=False)