def test_03_add_repo_package(self):
"""Verify that pkg(7) archive creation using add_repo_package()
works as expected.
"""
progtrack = pkg.client.progress.QuietProgressTracker()
# Get repository.
repo = self.get_repo(self.dc.get_repodir())
# Create an archive with just one package.
arc_path = os.path.join(self.test_root, "add_repo_package.p5p")
arc = pkg.p5p.Archive(arc_path, mode="w")
arc.add_repo_package(self.foo, repo)
arc.close(progtrack=progtrack)
# Verify the result.
arc = ptf.PkgTarFile(name=arc_path, mode="r")
expected = self.foo_expected
actual = [m.name for m in arc.getmembers()]
self.assertEqualDiff(expected, actual)
# Prep a new archive.
os.unlink(arc_path)
arc = pkg.p5p.Archive(arc_path, mode="w")
# Create an archive with multiple packages.
# (Don't use progtrack this time.)
arc.add_repo_package(self.foo, repo)
arc.add_repo_package(self.signed, repo)
arc.add_repo_package(self.quux, repo)
arc.close()
# Verify the result.
arc = ptf.PkgTarFile(name=arc_path, mode="r")
# Add in the p5i file since this is an archive with signed
# packages created from a repo.
expected = sorted(self.multi_expected + ["publisher/test/pub.p5i"])
action_certs = [
self.calc_pem_hash(t) for t in (
os.path.join(self.cs_dir, "cs1_ch5_ta1_cert.pem"),
os.path.join(self.chain_certs_dir, "ch1_ta1_cert.pem"),
os.path.join(self.chain_certs_dir, "ch2_ta1_cert.pem"),
os.path.join(self.chain_certs_dir, "ch3_ta1_cert.pem"),
os.path.join(self.chain_certs_dir, "ch4_ta1_cert.pem"),
os.path.join(self.chain_certs_dir, "ch5_ta1_cert.pem"),
os.path.join(self.chain_certs_dir, "ch1_ta3_cert.pem"),
)
]
for hsh in action_certs:
d = "publisher/test/file/{0}".format(hsh[0:2])
f = "{0}/{1}".format(d, hsh)
expected.append(d)
expected.append(f)
actual = sorted(m.name for m in arc.getmembers())
self.assertEqualDiff(sorted(set(expected)), actual)
os.unlink(arc_path)
def test_00_create(self):
"""Verify that archive creation works as expected."""
# Verify that an empty package archive can be created and that
# the resulting archive is of the correct type.
arc_path = os.path.join(self.test_root, "empty.p5p")
arc = pkg.p5p.Archive(arc_path, mode="w")
self.assertEqual(arc.pathname, arc_path)
arc.close()
# Verify archive exists and use the tarfile module to read the
# archive so that the implementation can be verified.
assert os.path.exists(arc_path)
arc = ptf.PkgTarFile(name=arc_path, mode="r")
fm = arc.firstmember
self.assertEqual(fm.name, "pkg5.index.0.gz")
comment = fm.pax_headers.get("comment", "")
self.assertEqual(comment, "pkg5.archive.version.0")
# Verify basic expected content exists.
expected = ["pkg5.index.0.gz", "publisher", "pkg5.repository"]
actual = [m.name for m in arc.getmembers()]
self.assertEqualDiff(expected, actual)
# Destroy the archive.
os.unlink(arc_path)
def testerrorlevelIsCorrect(self):
p = pkgtarfile.PkgTarFile(self.tarfile, 'r')
# "read-only" folders on Windows are not actually read-only so
# the test below doesn't cause the exception to be raised
if portable.is_admin() or portable.util.get_canonical_os_type(
) == "windows":
self.assert_(p.errorlevel == 2)
p.close()
return
extractpath = os.path.join(self.tpath, "foo/bar")
os.makedirs(extractpath)
os.chmod(extractpath, 0o555)
self.assertRaises(IOError, p.extract, "foo/bar/baz", self.tpath)
p.close()
os.chmod(extractpath, 0o777)
def test_05_invalid(self):
"""Verify that pkg(7) archive class handles broken archives
and items that aren't archives as expected."""
arc_path = os.path.join(self.test_root, "nosucharchive.p5p")
#
# Check that no archive is handled.
#
self.assertRaisesStringify(pkg.p5p.InvalidArchive,
pkg.p5p.Archive,
arc_path,
mode="r")
#
# Check that empty archive file is handled.
#
arc_path = os.path.join(self.test_root, "retrieve.p5p")
open(arc_path, "wb").close()
self.assertRaisesStringify(pkg.p5p.InvalidArchive,
pkg.p5p.Archive,
arc_path,
mode="r")
os.unlink(arc_path)
#
# Check that invalid archive file is handled.
#
with open(arc_path, "w") as f:
f.write("not_a_valid_archive")
self.assertRaisesStringify(pkg.p5p.InvalidArchive,
pkg.p5p.Archive,
arc_path,
mode="r")
os.unlink(arc_path)
#
# Check that a truncated archive is handled.
#
repo = self.get_repo(self.dc.get_repodir())
arc = pkg.p5p.Archive(arc_path, mode="w")
arc.add_repo_package(self.foo, repo)
arc.add_repo_package(self.signed, repo)
arc.add_repo_package(self.quux, repo)
arc.close()
#
# Check that truncated archives, or archives with invalid
# indexes are handled as expected.
#
# Determine where to truncate archive by looking for specific
# package file and then setting truncate location to halfway
# through data for file.
arc = ptf.PkgTarFile(name=arc_path, mode="r")
idx_data_offset = 0
src_offset = 0
src_bytes = 0
dest_offset = 0
trunc_sz = 0
src_fhash = "b265f2ec87c4a55eb2b6b4c926e7c65f7247a27e"
dest_fhash = "801eebbfe8c526bf092d98741d4228e4d0fc99ae"
for m in arc.getmembers():
if m.name.endswith("/" + dest_fhash):
dest_offset = m.offset
trunc_sz = m.offset_data + int(m.size // 2)
elif m.name.endswith("pkg5.index.0.gz"):
idx_data_offset = m.offset_data
elif m.name.endswith("/" + src_fhash):
# Calculate size of source entry.
src_bytes = m.offset_data - m.offset
blocks, rem = divmod(m.size, tf.BLOCKSIZE)
if rem > 0:
blocks += 1
src_bytes += blocks * tf.BLOCKSIZE
src_offset = m.offset
arc.close()
# Test truncated archive case.
bad_arc_path = os.path.join(self.test_root, "bad_arc.p5p")
portable.copyfile(arc_path, bad_arc_path)
self.debug("{0} size: {1:d} truncate: {2:d}".format(
arc_path,
os.stat(arc_path).st_size, trunc_sz))
with open(bad_arc_path, "ab+") as f:
f.truncate(trunc_sz)
ext_dir = os.path.join(self.test_root, "extracted")
shutil.rmtree(ext_dir, True)
arc = pkg.p5p.Archive(bad_arc_path, mode="r")
self.assertRaisesStringify(pkg.p5p.InvalidArchive,
arc.extract_package_files, [dest_fhash],
ext_dir,
pub="test2")
arc.close()
# Test archive with invalid index; do this by writing some bogus
# bytes into the data area for the index.
portable.copyfile(arc_path, bad_arc_path)
with open(bad_arc_path, "ab+") as dest:
dest.seek(idx_data_offset)
dest.truncate()
with open(arc_path, "rb") as src:
bogus_data = b"invalid_index_data"
dest.write(bogus_data)
src.seek(idx_data_offset + len(bogus_data))
dest.write(src.read())
shutil.rmtree(ext_dir, True)
self.assertRaisesStringify(pkg.p5p.InvalidArchive,
pkg.p5p.Archive,
bad_arc_path,
mode="r")
# Test archive with invalid index offsets; do this by truncating
# an existing archive at the offset of one of its files and then
# appending the data for a different archive member in its
# place.
portable.copyfile(arc_path, bad_arc_path)
with open(bad_arc_path, "ab+") as dest:
dest.seek(dest_offset)
dest.truncate()
with open(arc_path, "rb") as src:
src.seek(src_offset)
dest.write(src.read(src_bytes))
shutil.rmtree(ext_dir, True)
arc = pkg.p5p.Archive(bad_arc_path, mode="r")
self.assertRaisesStringify(pkg.p5p.InvalidArchive,
arc.extract_package_files, [dest_fhash],
ext_dir,
pub="test2")
arc.close()
os.unlink(arc_path)
os.unlink(bad_arc_path)
#
# Check that directory where archive expected is handled.
#
os.mkdir(arc_path)
self.assertRaisesStringify(pkg.p5p.InvalidArchive,
pkg.p5p.Archive,
arc_path,
mode="r")
os.rmdir(arc_path)
# Temporarily change the current archive version and create a
# a new archive, and then verify that the expected exception is
# raised when an attempt to read it is made.
orig_ver = pkg.p5p.Archive.CURRENT_VERSION
try:
pkg.p5p.Archive.CURRENT_VERSION = 99 # EVIL
arc = pkg.p5p.Archive(arc_path, mode="w")
arc.close()
finally:
# Ensure this is reset to the right value.
pkg.p5p.Archive.CURRENT_VERSION = orig_ver
self.assertRaisesStringify(pkg.p5p.InvalidArchive,
pkg.p5p.Archive,
arc_path,
mode="r")
os.unlink(arc_path)
def test_04_extract(self):
"""Verify that pkg(7) archive extraction methods work as
expected.
"""
# Get repository.
repo = self.get_repo(self.dc.get_repodir())
# Create an archive with a few packages.
arc_path = os.path.join(self.test_root, "retrieve.p5p")
arc = pkg.p5p.Archive(arc_path, mode="w")
arc.add_repo_package(self.foo, repo)
arc.add_repo_package(self.signed, repo)
arc.add_repo_package(self.quux, repo)
arc.close()
# Get list of file hashes. These will be the "least-preferred"
# hash for the actions being stored.
hashes = {"all": set()}
for rstore in repo.rstores:
for dirpath, dirnames, filenames in os.walk(rstore.file_root):
if not filenames:
continue
hashes["all"].update(filenames)
hashes.setdefault(rstore.publisher, set()).update(filenames)
# Extraction directory for testing.
ext_dir = os.path.join(self.test_root, "extracted")
# First, verify behaviour using archive created using
# pkg(7) archive class.
arc = self.__verify_extract(repo, arc_path, hashes, ext_dir)
arc.close()
# Now extract everything from the archive and create
# a new archive using the tarfile class, and verify
# that the pkg(7) archive class can still extract
# and access the contents as expected even though
# the index file isn't marked with the appropriate
# pax headers (and so should be ignored since it's
# also invalid).
shutil.rmtree(ext_dir)
# Extract all of the existing content.
arc = ptf.PkgTarFile(name=arc_path, mode="r")
arc.extractall(ext_dir)
arc.close()
# Create a new archive.
os.unlink(arc_path)
arc = ptf.PkgTarFile(name=arc_path, mode="w")
def add_entry(src):
fpath = os.path.join(dirpath, src)
arcname = pkg.misc.relpath(fpath, ext_dir)
arc.add(name=fpath, arcname=arcname, recursive=False)
for dirpath, dirnames, filenames in os.walk(ext_dir):
list(map(add_entry, filenames))
list(map(add_entry, dirnames))
arc.close()
# Verify that archive has expected contents.
arc = ptf.PkgTarFile(name=arc_path, mode="r")
# Add in the p5i file since this is an archive with signed
# packages created from a repo.
expected = sorted(self.multi_expected + ["publisher/test/pub.p5i"])
action_certs = [
self.calc_pem_hash(t) for t in (
os.path.join(self.cs_dir, "cs1_ch5_ta1_cert.pem"),
os.path.join(self.chain_certs_dir, "ch1_ta1_cert.pem"),
os.path.join(self.chain_certs_dir, "ch2_ta1_cert.pem"),
os.path.join(self.chain_certs_dir, "ch3_ta1_cert.pem"),
os.path.join(self.chain_certs_dir, "ch4_ta1_cert.pem"),
os.path.join(self.chain_certs_dir, "ch5_ta1_cert.pem"),
os.path.join(self.chain_certs_dir, "ch1_ta3_cert.pem"),
)
]
for hsh in action_certs:
d = "publisher/test/file/{0}".format(hsh[0:2])
f = "{0}/{1}".format(d, hsh)
expected.append(d)
expected.append(f)
actual = sorted(m.name for m in arc.getmembers())
self.assertEqualDiff(sorted(set(expected)), actual)
arc.close()
# Verify pkg(7) archive class extraction behaviour using
# the new archive.
arc = self.__verify_extract(repo, arc_path, hashes, ext_dir)
arc.close()
# Extract all of the existing content.
arc = ptf.PkgTarFile(name=arc_path, mode="r")
arc.extractall(ext_dir)
arc.close()
# Now verify archive can still be used when index file
# is omitted.
os.unlink(arc_path)
arc = ptf.PkgTarFile(name=arc_path, mode="w")
for dirpath, dirnames, filenames in os.walk(ext_dir):
list(
map(add_entry,
[f for f in filenames if f != "pkg5.index.0.gz"]))
list(map(add_entry, dirnames))
arc.close()
# Verify pkg(7) archive class extraction behaviour using
# the new archive.
arc = self.__verify_extract(repo, arc_path, hashes, ext_dir)
arc.close()
# Save an index for later.
arc = pkg.p5p.Archive(arc_path, mode="r")
saved_index = arc.get_index()
arc.close()
# Verify we can extract the archive reusing an index.
arc = self.__verify_extract(repo,
arc_path,
hashes,
ext_dir,
archive_index=saved_index)
arc.close()
# Verify we throw an assert when opening a p5p in write mode.
self.assertRaisesStringify(AssertionError,
pkg.p5p.Archive,
arc_path,
mode="w",
archive_index=saved_index)
# Verify we can't extract archive members using a corrupted
# index.
arc = pkg.p5p.Archive(arc_path, mode="r", archive_index={"cats": 1234})
self.assertRaisesStringify(pkg.p5p.ArchiveErrors, arc.extract_catalog1,
"catalog.attrs", ext_dir)
self.assertRaisesStringify(pkg.p5p.ArchiveErrors,
arc.extract_package_files, hashes, ext_dir)
arc.close()
def test_02_add_package(self):
"""Verify that pkg(7) archive creation using add_package() works
as expected.
"""
# Get repository.
repo = self.get_repo(self.dc.get_repodir())
# Create a directory and copy package files from repository to
# it (this is how pkgrecv stores content during republication
# or when using --raw).
dfroot = os.path.join(self.test_root, "pfiles")
os.mkdir(dfroot, pkg.misc.PKG_DIR_MODE)
foo_path = os.path.join(dfroot, "foo.p5m")
portable.copyfile(repo.manifest(self.foo), foo_path)
signed_path = os.path.join(dfroot, "signed.p5m")
portable.copyfile(repo.manifest(self.signed), signed_path)
quux_path = os.path.join(dfroot, "quux.p5m")
portable.copyfile(repo.manifest(self.quux), quux_path)
for rstore in repo.rstores:
for dirpath, dirnames, filenames in os.walk(rstore.file_root):
if not filenames:
continue
for f in filenames:
portable.copyfile(os.path.join(dirpath, f),
os.path.join(dfroot, f))
# Prep the archive.
progtrack = pkg.client.progress.QuietProgressTracker()
arc_path = os.path.join(self.test_root, "add_package.p5p")
arc = pkg.p5p.Archive(arc_path, mode="w")
# Create an archive with just one package.
arc.add_package(self.foo, foo_path, dfroot)
arc.close(progtrack=progtrack)
# Verify the result.
arc = ptf.PkgTarFile(name=arc_path, mode="r")
expected = self.foo_expected
actual = [m.name for m in arc.getmembers()]
self.assertEqualDiff(expected, actual)
# Prep a new archive.
os.unlink(arc_path)
arc = pkg.p5p.Archive(arc_path, mode="w")
# Create an archive with multiple packages.
# (Don't use progtrack this time.)
arc.add_package(self.foo, foo_path, dfroot)
arc.add_package(self.signed, signed_path, dfroot)
arc.add_package(self.quux, quux_path, dfroot)
arc.close()
# Verify the result.
arc = ptf.PkgTarFile(name=arc_path, mode="r")
expected = self.multi_expected[:]
action_certs = [
self.calc_pem_hash(t) for t in (
os.path.join(self.cs_dir, "cs1_ch5_ta1_cert.pem"),
os.path.join(self.chain_certs_dir, "ch1_ta1_cert.pem"),
os.path.join(self.chain_certs_dir, "ch2_ta1_cert.pem"),
os.path.join(self.chain_certs_dir, "ch3_ta1_cert.pem"),
os.path.join(self.chain_certs_dir, "ch4_ta1_cert.pem"),
os.path.join(self.chain_certs_dir, "ch5_ta1_cert.pem"),
os.path.join(self.chain_certs_dir, "ch1_ta3_cert.pem"),
)
]
for hsh in action_certs:
d = "publisher/test/file/{0}".format(hsh[0:2])
f = "{0}/{1}".format(d, hsh)
expected.append(d)
expected.append(f)
actual = sorted(m.name for m in arc.getmembers())
self.assertEqualDiff(sorted(set(expected)), actual)
os.unlink(arc_path)
os.unlink(foo_path)
os.unlink(quux_path)
os.unlink(signed_path)
def test_01_add(self):
"""Verify that add() works as expected."""
# Prep the archive.
arc_path = os.path.join(self.test_root, "add.p5p")
arc = pkg.p5p.Archive(arc_path, mode="w")
# add() permits addition of arbitrary files (intentionally);
# it is also the routine that higher-level functions to add
# package content use internally. Because of that, this
# function does not strictly need standalone testing, but it
# helps ensure all code paths for add() are tested.
arc.add(self.test_root)
tmp_root = os.path.join(self.test_root, "tmp")
arc.add(tmp_root)
for f in self.misc_files:
src = os.path.join(self.test_root, f)
# Ensure files are read-only mode so that file perm
# normalization can be tested.
os.chmod(src, pkg.misc.PKG_RO_FILE_MODE)
arc.add(src)
# Write out archive.
arc.close()
# Now open the archive and iterate through its contents and
# verify that each member has the expected characteristics.
arc = ptf.PkgTarFile(name=arc_path, mode="r")
members = [m for m in arc.getmembers()]
# Should be 11 files including package archive index and three
# directories.
actual = [m.name for m in members]
self.assertEqual(len(actual), 11)
expected = [
"pkg5.index.0.gz", "publisher",
pkg.misc.relpath(self.test_root, "/"),
pkg.misc.relpath(tmp_root, "/")
]
expected.extend(
pkg.misc.relpath(os.path.join(self.test_root, e), "/")
for e in self.misc_files)
expected.append("pkg5.repository")
self.assertEqualDiff(expected, actual)
for member in members:
# All archive members should be a file or directory.
self.assertTrue(member.isreg() or member.isdir())
if member.name == "pkg5.index.0.gz":
assert member.isreg()
comment = member.pax_headers.get("comment", "")
self.assertEqual(comment, "pkg5.archive.version.0")
continue
if member.isdir():
# Verify directories were added with expected
# mode.
self.assertEqual(oct(member.mode), oct(pkg.misc.PKG_DIR_MODE))
elif member.isfile():
# Verify files were added with expected mode.
self.assertEqual(oct(member.mode), oct(pkg.misc.PKG_FILE_MODE))
# Verify files and directories have expected ownership.
self.assertEqual(member.uname, "root")
self.assertEqual(member.gname, "root")
self.assertEqual(member.uid, 0)
self.assertEqual(member.gid, 0)
os.unlink(arc_path)
