def test_tar(self):
# TODO: test XZCompressedTarArchiver? Needs an xz BinaryTool, so hard to see how to do in a
# unit test.
self.round_trip(create_archiver('tar'),
expected_ext='tar',
empty_dirs=True)
self.round_trip(create_archiver('tgz'),
expected_ext='tar.gz',
empty_dirs=True)
self.round_trip(create_archiver('tbz2'),
expected_ext='tar.bz2',
empty_dirs=True)
def test_tar(self):
self.round_trip(create_archiver("tar"),
expected_ext="tar",
empty_dirs=True)
self.round_trip(create_archiver("tgz"),
expected_ext="tar.gz",
empty_dirs=True)
self.round_trip(create_archiver("tbz2"),
expected_ext="tar.bz2",
empty_dirs=True)
self.round_trip(create_archiver("txz"),
expected_ext="tar.xz",
empty_dirs=True)
def test_zip_filter(self):
def do_filter(path):
return path == 'allowed.txt'
with temporary_dir() as fromdir:
touch(os.path.join(fromdir, 'allowed.txt'))
touch(os.path.join(fromdir, 'disallowed.txt'))
with temporary_dir() as archivedir:
archive = create_archiver('zip').create(fromdir, archivedir, 'archive')
with temporary_dir() as todir:
create_archiver('zip').extract(archive, todir, filter_func=do_filter)
self.assertEqual(set(['allowed.txt']), self._listtree(todir, empty_dirs=False))
def test_zip_filter(self):
def do_filter(path):
return path == 'allowed.txt'
with temporary_dir() as fromdir:
touch(os.path.join(fromdir, 'allowed.txt'))
touch(os.path.join(fromdir, 'disallowed.txt'))
with temporary_dir() as archivedir:
archive = create_archiver('zip').create(fromdir, archivedir, 'archive')
with temporary_dir() as todir:
create_archiver('zip').extract(archive, todir, filter_func=do_filter)
self.assertEquals(set(['allowed.txt']), self._listtree(todir, empty_dirs=False))
def check_archive_with_flags(archive_format, dereference):
with temporary_dir() as fromdir:
filename = os.path.join(fromdir, 'a')
linkname = os.path.join(fromdir, 'link_to_a')
touch(filename)
relative_symlink(filename, linkname)
with temporary_dir() as archivedir:
archive = create_archiver(archive_format).create(fromdir, archivedir, 'archive', dereference=dereference)
with temporary_dir() as todir:
create_archiver(archive_format).extract(archive, todir)
extracted_linkname = os.path.join(todir, 'link_to_a')
assertion = self.assertFalse if dereference else self.assertTrue
assertion(os.path.islink(extracted_linkname))
assertion(os.path.samefile(extracted_linkname, os.path.join(todir, 'a')))
def check_archive_with_flags(archive_format, dereference):
with temporary_dir() as fromdir:
filename = os.path.join(fromdir, 'a')
linkname = os.path.join(fromdir, 'link_to_a')
touch(filename)
relative_symlink(filename, linkname)
with temporary_dir() as archivedir:
archive = create_archiver(archive_format).create(fromdir, archivedir, 'archive', dereference=dereference)
with temporary_dir() as todir:
create_archiver(archive_format).extract(archive, todir)
extracted_linkname = os.path.join(todir, 'link_to_a')
assertion = self.assertFalse if dereference else self.assertTrue
assertion(os.path.islink(extracted_linkname))
assertion(os.path.samefile(extracted_linkname, os.path.join(todir, 'a')))
def execute(self):
targets_to_bundle = self.context.targets(PythonApp.is_python_app)
with self.invalidated(targets_to_bundle, invalidate_dependents=True) as invalidation_check:
bundle_archive_product = self.context.products.get(self._DEPLOYABLE_ARCHIVES)
for vt in invalidation_check.all_vts:
bundle_dir = self.get_bundle_dir(vt.target.id, vt.results_dir)
archive_format = self.resolved_option(self.get_options(), vt.target, 'archive')
archiver = archive.create_archiver(archive_format) if archive_format else None
archive_path = self._get_archive_path(vt, archive_format)
if not vt.valid: # Only recreate the bundle/archive if it's changed
self._bundle(vt.target, bundle_dir)
if archiver:
archiver.create(bundle_dir, vt.results_dir, vt.target.id)
self.context.log.info(
'created archive {}'.format(os.path.relpath(archive_path, get_buildroot())))
if archiver:
bundle_archive_product.add(
vt.target, os.path.dirname(archive_path)).append(os.path.basename(archive_path))
if vt.target in self.context.target_roots: # Always publish bundle/archive in dist
self.publish_results(self.get_options().pants_distdir,
False,
vt,
bundle_dir,
archive_path,
vt.target.id,
archive_format)
def execute(self):
targets_to_bundle = self.context.targets(PythonApp.is_python_app)
with self.invalidated(
targets_to_bundle,
invalidate_dependents=True) as invalidation_check:
bundle_archive_product = self.context.products.get(
self._DEPLOYABLE_ARCHIVES)
for vt in invalidation_check.all_vts:
bundle_dir = self.get_bundle_dir(vt.target.id, vt.results_dir)
archive_format = self.resolved_option(self.get_options(),
vt.target, 'archive')
archiver = archive.create_archiver(
archive_format) if archive_format else None
archive_path = self._get_archive_path(vt, archive_format)
if not vt.valid: # Only recreate the bundle/archive if it's changed
self._bundle(vt.target, bundle_dir)
if archiver:
archiver.create(bundle_dir, vt.results_dir,
vt.target.id)
self.context.log.info('created archive {}'.format(
os.path.relpath(archive_path, get_buildroot())))
if archiver:
bundle_archive_product.add(
vt.target, os.path.dirname(archive_path)).append(
os.path.basename(archive_path))
if vt.target in self.context.target_roots: # Always publish bundle/archive in dist
self.publish_results(self.get_options().pants_distdir,
False, vt, bundle_dir, archive_path,
vt.target.id, archive_format)
def execute(self):
bundleable_js = self.context.products.get_data('bundleable_js')
bundle_archive_product = self.context.products.get('deployable_archives')
dirutil.safe_mkdir(self._outdir) # Make sure dist dir is present.
for target in self.context.target_roots:
if self.is_node_bundle(target):
archiver = archive.create_archiver(target.payload.archive)
for _, abs_paths in bundleable_js[target.node_module].abs_paths():
for abs_path in abs_paths:
# build_dir is a symlink. Since dereference option for tar is set to False, we need to
# dereference manually to archive the linked build dir.
build_dir = os.path.realpath(abs_path)
self.context.log.debug('archiving {}'.format(build_dir))
archivepath = archiver.create(
build_dir,
self._outdir,
target.package_name,
prefix=None,
dereference=False
)
bundle_archive_product.add(
target, os.path.dirname(archivepath)).append(os.path.basename(archivepath))
self.context.log.info(
'created {}'.format(os.path.relpath(archivepath, get_buildroot())))
def execute(self):
bundleable_js = self.context.products.get_data('bundleable_js')
bundle_archive_product = self.context.products.get(
'deployable_archives')
dirutil.safe_mkdir(self._outdir) # Make sure dist dir is present.
for target in self.context.target_roots:
if self.is_node_bundle(target):
archiver = archive.create_archiver(target.payload.archive)
for _, abs_paths in bundleable_js[
target.node_module].abs_paths():
for abs_path in abs_paths:
# build_dir is a symlink. Since dereference option for tar is set to False, we need to
# dereference manually to archive the linked build dir.
build_dir = os.path.realpath(abs_path)
self.context.log.debug(
'archiving {}'.format(build_dir))
archivepath = archiver.create(build_dir,
self._outdir,
target.package_name,
prefix=None,
dereference=False)
bundle_archive_product.add(
target, os.path.dirname(archivepath)).append(
os.path.basename(archivepath))
self.context.log.info('created {}'.format(
os.path.relpath(archivepath, get_buildroot())))
def test_zip_filter(self):
def do_filter(path):
return path == "allowed.txt"
with temporary_dir() as fromdir:
touch(os.path.join(fromdir, "allowed.txt"))
touch(os.path.join(fromdir, "disallowed.txt"))
with temporary_dir() as archivedir:
archive = create_archiver("zip").create(
fromdir, archivedir, "archive")
with temporary_dir() as todir:
create_archiver("zip").extract(archive,
todir,
filter_func=do_filter)
self.assertEqual({"allowed.txt"},
self._listtree(todir, empty_dirs=False))
def _get_archiver(self):
if not self.archive_type:
return None
if self.archive_type == 'txz':
return self._xz.tar_xz_extractor
return create_archiver(self.archive_type)
def _get_archiver(self):
if not self.archive_type:
return None
if self.archive_type == 'txz':
return self._xz.tar_xz_extractor
return create_archiver(self.archive_type)
def select(self, supportdir, version, name, platform_dependent,
archive_type):
"""Fetches a file, unpacking it if necessary."""
if archive_type is None:
return self._select_file(supportdir, version, name,
platform_dependent)
archiver = create_archiver(archive_type)
return self._select_archive(supportdir, version, name,
platform_dependent, archiver)
def _extract_archive(self, archive_path):
with temporary_dir() as temp_dir:
_, extension = os.path.splitext(archive_path)
print(extension)
if extension == '.jar':
extraction_archiver = create_archiver('zip')
else:
extraction_archiver = archiver_for_path(os.path.basename(archive_path))
extraction_archiver.extract(archive_path, temp_dir)
yield temp_dir
def _extract_archive(self, archive_path):
with temporary_dir() as temp_dir:
_, extension = os.path.splitext(archive_path)
print(extension)
if extension == '.jar':
extraction_archiver = create_archiver('zip')
else:
extraction_archiver = archiver_for_path(os.path.basename(archive_path))
extraction_archiver.extract(archive_path, temp_dir)
yield temp_dir
def _get_archiver(self):
if not self.archive_type:
return None
# This forces downloading and extracting the `XZ` archive if any BinaryTool with a 'txz'
# archive_type is used, but that's fine, because unless the cache is manually changed we won't
# do more work than necessary.
if self.archive_type == 'txz':
return self._xz.tar_xz_extractor
return create_archiver(self.archive_type)
def _get_archiver(self):
if not self.archive_type:
return None
# This forces downloading and extracting the `XZ` archive if any BinaryTool with a 'txz'
# archive_type is used, but that's fine, because unless the cache is manually changed we won't
# do more work than necessary.
if self.archive_type == "txz":
return self._xz.tar_xz_extractor
return create_archiver(self.archive_type)
def execute(self):
targets_to_bundle = self.context.targets(self.App.is_app)
if self.get_options().use_basename_prefix:
self.check_basename_conflicts([
t for t in self.context.target_roots if t in targets_to_bundle
])
with self.invalidated(
targets_to_bundle,
invalidate_dependents=True) as invalidation_check:
jvm_bundles_product = self.context.products.get("jvm_bundles")
bundle_archive_product = self.context.products.get(
"deployable_archives")
jvm_archive_product = self.context.products.get("jvm_archives")
for vt in invalidation_check.all_vts:
app = self.App.create_app(
vt.target,
self.resolved_option(self.get_options(), vt.target,
"deployjar"),
self.resolved_option(self.get_options(), vt.target,
"archive"),
)
archiver = archive.create_archiver(
app.archive) if app.archive else None
bundle_dir = self.get_bundle_dir(app.id, vt.results_dir)
ext = archive.archive_extensions.get(app.archive, app.archive)
filename = f"{app.id}.{ext}"
archive_path = os.path.join(vt.results_dir,
filename) if app.archive else ""
if not vt.valid:
self.bundle(app, vt.results_dir)
if app.archive:
archiver.create(bundle_dir, vt.results_dir, app.id)
self._add_product(jvm_bundles_product, app, bundle_dir)
if archiver:
self._add_product(bundle_archive_product, app,
archive_path)
self._add_product(jvm_archive_product, app, archive_path)
# For root targets, create symlink.
if vt.target in self.context.target_roots:
self.publish_results(
self.get_options().pants_distdir,
self.get_options().use_basename_prefix,
vt,
bundle_dir,
archive_path,
app.id,
app.archive,
)
def execute(self):
targets_to_bundle = self.context.targets(self.App.is_app)
if self.get_options().use_basename_prefix:
self.check_basename_conflicts([t for t in self.context.target_roots if t in targets_to_bundle])
with self.invalidated(targets_to_bundle, invalidate_dependents=True) as invalidation_check:
jvm_bundles_product = self.context.products.get('jvm_bundles')
bundle_archive_product = self.context.products.get('deployable_archives')
jvm_archive_product = self.context.products.get('jvm_archives')
for vt in invalidation_check.all_vts:
app = self.App.create_app(vt.target,
self.resolved_option(self.get_options(), vt.target, 'deployjar'),
self.resolved_option(self.get_options(), vt.target, 'archive'))
archiver = archive.create_archiver(app.archive) if app.archive else None
bundle_dir = self.get_bundle_dir(app.id, vt.results_dir)
ext = archive.archive_extensions.get(app.archive, app.archive)
filename = '{}.{}'.format(app.id, ext)
archive_path = os.path.join(vt.results_dir, filename) if app.archive else ''
if not vt.valid:
self.bundle(app, vt.results_dir)
if app.archive:
archiver.create(bundle_dir, vt.results_dir, app.id)
self._add_product(jvm_bundles_product, app, bundle_dir)
if archiver:
self._add_product(bundle_archive_product, app, archive_path)
self._add_product(jvm_archive_product, app, archive_path)
# For root targets, create symlink.
if vt.target in self.context.target_roots:
self.publish_results(self.get_options().pants_distdir,
self.get_options().use_basename_prefix,
vt,
bundle_dir,
archive_path,
app.id,
app.archive)
def test_zip(self):
self.round_trip(create_archiver("zip"),
expected_ext="zip",
empty_dirs=False)
def select(self, supportdir, version, name, platform_dependent, archive_type):
"""Fetches a file, unpacking it if necessary."""
if archive_type is None:
return self._select_file(supportdir, version, name, platform_dependent)
archiver = create_archiver(archive_type)
return self._select_archive(supportdir, version, name, platform_dependent, archiver)
def _get_archiver(self):
if not self.archive_type:
return None
return create_archiver(self.archive_type)
def test_zip(self):
self.round_trip(create_archiver('zip'),
expected_ext='zip',
empty_dirs=False)
def test_tar(self):
# TODO: test XZCompressedTarArchiver? Needs an xz BinaryTool, so hard to see how to do in a
# unit test.
self.round_trip(create_archiver('tar'), expected_ext='tar', empty_dirs=True)
self.round_trip(create_archiver('tgz'), expected_ext='tar.gz', empty_dirs=True)
self.round_trip(create_archiver('tbz2'), expected_ext='tar.bz2', empty_dirs=True)
def test_zip(self):
self.round_trip(create_archiver('zip'), expected_ext='zip', empty_dirs=False)
