def test_03_add_repo_package(self): """Verify that pkg(7) archive creation using add_repo_package() works as expected. """ progtrack = pkg.client.progress.QuietProgressTracker() # Get repository. repo = self.get_repo(self.dc.get_repodir()) # Create an archive with just one package. arc_path = os.path.join(self.test_root, "add_repo_package.p5p") arc = pkg.p5p.Archive(arc_path, mode="w") arc.add_repo_package(self.foo, repo) arc.close(progtrack=progtrack) # Verify the result. arc = ptf.PkgTarFile(name=arc_path, mode="r") expected = self.foo_expected actual = [m.name for m in arc.getmembers()] self.assertEqualDiff(expected, actual) # Prep a new archive. os.unlink(arc_path) arc = pkg.p5p.Archive(arc_path, mode="w") # Create an archive with multiple packages. # (Don't use progtrack this time.) arc.add_repo_package(self.foo, repo) arc.add_repo_package(self.signed, repo) arc.add_repo_package(self.quux, repo) arc.close() # Verify the result. arc = ptf.PkgTarFile(name=arc_path, mode="r") # Add in the p5i file since this is an archive with signed # packages created from a repo. expected = sorted(self.multi_expected + ["publisher/test/pub.p5i"]) action_certs = [ self.calc_pem_hash(t) for t in ( os.path.join(self.cs_dir, "cs1_ch5_ta1_cert.pem"), os.path.join(self.chain_certs_dir, "ch1_ta1_cert.pem"), os.path.join(self.chain_certs_dir, "ch2_ta1_cert.pem"), os.path.join(self.chain_certs_dir, "ch3_ta1_cert.pem"), os.path.join(self.chain_certs_dir, "ch4_ta1_cert.pem"), os.path.join(self.chain_certs_dir, "ch5_ta1_cert.pem"), os.path.join(self.chain_certs_dir, "ch1_ta3_cert.pem"), ) ] for hsh in action_certs: d = "publisher/test/file/{0}".format(hsh[0:2]) f = "{0}/{1}".format(d, hsh) expected.append(d) expected.append(f) actual = sorted(m.name for m in arc.getmembers()) self.assertEqualDiff(sorted(set(expected)), actual) os.unlink(arc_path)
def test_00_create(self): """Verify that archive creation works as expected.""" # Verify that an empty package archive can be created and that # the resulting archive is of the correct type. arc_path = os.path.join(self.test_root, "empty.p5p") arc = pkg.p5p.Archive(arc_path, mode="w") self.assertEqual(arc.pathname, arc_path) arc.close() # Verify archive exists and use the tarfile module to read the # archive so that the implementation can be verified. assert os.path.exists(arc_path) arc = ptf.PkgTarFile(name=arc_path, mode="r") fm = arc.firstmember self.assertEqual(fm.name, "pkg5.index.0.gz") comment = fm.pax_headers.get("comment", "") self.assertEqual(comment, "pkg5.archive.version.0") # Verify basic expected content exists. expected = ["pkg5.index.0.gz", "publisher", "pkg5.repository"] actual = [m.name for m in arc.getmembers()] self.assertEqualDiff(expected, actual) # Destroy the archive. os.unlink(arc_path)
def testerrorlevelIsCorrect(self): p = pkgtarfile.PkgTarFile(self.tarfile, 'r') # "read-only" folders on Windows are not actually read-only so # the test below doesn't cause the exception to be raised if portable.is_admin() or portable.util.get_canonical_os_type( ) == "windows": self.assert_(p.errorlevel == 2) p.close() return extractpath = os.path.join(self.tpath, "foo/bar") os.makedirs(extractpath) os.chmod(extractpath, 0o555) self.assertRaises(IOError, p.extract, "foo/bar/baz", self.tpath) p.close() os.chmod(extractpath, 0o777)
def test_05_invalid(self): """Verify that pkg(7) archive class handles broken archives and items that aren't archives as expected.""" arc_path = os.path.join(self.test_root, "nosucharchive.p5p") # # Check that no archive is handled. # self.assertRaisesStringify(pkg.p5p.InvalidArchive, pkg.p5p.Archive, arc_path, mode="r") # # Check that empty archive file is handled. # arc_path = os.path.join(self.test_root, "retrieve.p5p") open(arc_path, "wb").close() self.assertRaisesStringify(pkg.p5p.InvalidArchive, pkg.p5p.Archive, arc_path, mode="r") os.unlink(arc_path) # # Check that invalid archive file is handled. # with open(arc_path, "w") as f: f.write("not_a_valid_archive") self.assertRaisesStringify(pkg.p5p.InvalidArchive, pkg.p5p.Archive, arc_path, mode="r") os.unlink(arc_path) # # Check that a truncated archive is handled. # repo = self.get_repo(self.dc.get_repodir()) arc = pkg.p5p.Archive(arc_path, mode="w") arc.add_repo_package(self.foo, repo) arc.add_repo_package(self.signed, repo) arc.add_repo_package(self.quux, repo) arc.close() # # Check that truncated archives, or archives with invalid # indexes are handled as expected. # # Determine where to truncate archive by looking for specific # package file and then setting truncate location to halfway # through data for file. arc = ptf.PkgTarFile(name=arc_path, mode="r") idx_data_offset = 0 src_offset = 0 src_bytes = 0 dest_offset = 0 trunc_sz = 0 src_fhash = "b265f2ec87c4a55eb2b6b4c926e7c65f7247a27e" dest_fhash = "801eebbfe8c526bf092d98741d4228e4d0fc99ae" for m in arc.getmembers(): if m.name.endswith("/" + dest_fhash): dest_offset = m.offset trunc_sz = m.offset_data + int(m.size // 2) elif m.name.endswith("pkg5.index.0.gz"): idx_data_offset = m.offset_data elif m.name.endswith("/" + src_fhash): # Calculate size of source entry. src_bytes = m.offset_data - m.offset blocks, rem = divmod(m.size, tf.BLOCKSIZE) if rem > 0: blocks += 1 src_bytes += blocks * tf.BLOCKSIZE src_offset = m.offset arc.close() # Test truncated archive case. bad_arc_path = os.path.join(self.test_root, "bad_arc.p5p") portable.copyfile(arc_path, bad_arc_path) self.debug("{0} size: {1:d} truncate: {2:d}".format( arc_path, os.stat(arc_path).st_size, trunc_sz)) with open(bad_arc_path, "ab+") as f: f.truncate(trunc_sz) ext_dir = os.path.join(self.test_root, "extracted") shutil.rmtree(ext_dir, True) arc = pkg.p5p.Archive(bad_arc_path, mode="r") self.assertRaisesStringify(pkg.p5p.InvalidArchive, arc.extract_package_files, [dest_fhash], ext_dir, pub="test2") arc.close() # Test archive with invalid index; do this by writing some bogus # bytes into the data area for the index. portable.copyfile(arc_path, bad_arc_path) with open(bad_arc_path, "ab+") as dest: dest.seek(idx_data_offset) dest.truncate() with open(arc_path, "rb") as src: bogus_data = b"invalid_index_data" dest.write(bogus_data) src.seek(idx_data_offset + len(bogus_data)) dest.write(src.read()) shutil.rmtree(ext_dir, True) self.assertRaisesStringify(pkg.p5p.InvalidArchive, pkg.p5p.Archive, bad_arc_path, mode="r") # Test archive with invalid index offsets; do this by truncating # an existing archive at the offset of one of its files and then # appending the data for a different archive member in its # place. portable.copyfile(arc_path, bad_arc_path) with open(bad_arc_path, "ab+") as dest: dest.seek(dest_offset) dest.truncate() with open(arc_path, "rb") as src: src.seek(src_offset) dest.write(src.read(src_bytes)) shutil.rmtree(ext_dir, True) arc = pkg.p5p.Archive(bad_arc_path, mode="r") self.assertRaisesStringify(pkg.p5p.InvalidArchive, arc.extract_package_files, [dest_fhash], ext_dir, pub="test2") arc.close() os.unlink(arc_path) os.unlink(bad_arc_path) # # Check that directory where archive expected is handled. # os.mkdir(arc_path) self.assertRaisesStringify(pkg.p5p.InvalidArchive, pkg.p5p.Archive, arc_path, mode="r") os.rmdir(arc_path) # Temporarily change the current archive version and create a # a new archive, and then verify that the expected exception is # raised when an attempt to read it is made. orig_ver = pkg.p5p.Archive.CURRENT_VERSION try: pkg.p5p.Archive.CURRENT_VERSION = 99 # EVIL arc = pkg.p5p.Archive(arc_path, mode="w") arc.close() finally: # Ensure this is reset to the right value. pkg.p5p.Archive.CURRENT_VERSION = orig_ver self.assertRaisesStringify(pkg.p5p.InvalidArchive, pkg.p5p.Archive, arc_path, mode="r") os.unlink(arc_path)
def test_04_extract(self): """Verify that pkg(7) archive extraction methods work as expected. """ # Get repository. repo = self.get_repo(self.dc.get_repodir()) # Create an archive with a few packages. arc_path = os.path.join(self.test_root, "retrieve.p5p") arc = pkg.p5p.Archive(arc_path, mode="w") arc.add_repo_package(self.foo, repo) arc.add_repo_package(self.signed, repo) arc.add_repo_package(self.quux, repo) arc.close() # Get list of file hashes. These will be the "least-preferred" # hash for the actions being stored. hashes = {"all": set()} for rstore in repo.rstores: for dirpath, dirnames, filenames in os.walk(rstore.file_root): if not filenames: continue hashes["all"].update(filenames) hashes.setdefault(rstore.publisher, set()).update(filenames) # Extraction directory for testing. ext_dir = os.path.join(self.test_root, "extracted") # First, verify behaviour using archive created using # pkg(7) archive class. arc = self.__verify_extract(repo, arc_path, hashes, ext_dir) arc.close() # Now extract everything from the archive and create # a new archive using the tarfile class, and verify # that the pkg(7) archive class can still extract # and access the contents as expected even though # the index file isn't marked with the appropriate # pax headers (and so should be ignored since it's # also invalid). shutil.rmtree(ext_dir) # Extract all of the existing content. arc = ptf.PkgTarFile(name=arc_path, mode="r") arc.extractall(ext_dir) arc.close() # Create a new archive. os.unlink(arc_path) arc = ptf.PkgTarFile(name=arc_path, mode="w") def add_entry(src): fpath = os.path.join(dirpath, src) arcname = pkg.misc.relpath(fpath, ext_dir) arc.add(name=fpath, arcname=arcname, recursive=False) for dirpath, dirnames, filenames in os.walk(ext_dir): list(map(add_entry, filenames)) list(map(add_entry, dirnames)) arc.close() # Verify that archive has expected contents. arc = ptf.PkgTarFile(name=arc_path, mode="r") # Add in the p5i file since this is an archive with signed # packages created from a repo. expected = sorted(self.multi_expected + ["publisher/test/pub.p5i"]) action_certs = [ self.calc_pem_hash(t) for t in ( os.path.join(self.cs_dir, "cs1_ch5_ta1_cert.pem"), os.path.join(self.chain_certs_dir, "ch1_ta1_cert.pem"), os.path.join(self.chain_certs_dir, "ch2_ta1_cert.pem"), os.path.join(self.chain_certs_dir, "ch3_ta1_cert.pem"), os.path.join(self.chain_certs_dir, "ch4_ta1_cert.pem"), os.path.join(self.chain_certs_dir, "ch5_ta1_cert.pem"), os.path.join(self.chain_certs_dir, "ch1_ta3_cert.pem"), ) ] for hsh in action_certs: d = "publisher/test/file/{0}".format(hsh[0:2]) f = "{0}/{1}".format(d, hsh) expected.append(d) expected.append(f) actual = sorted(m.name for m in arc.getmembers()) self.assertEqualDiff(sorted(set(expected)), actual) arc.close() # Verify pkg(7) archive class extraction behaviour using # the new archive. arc = self.__verify_extract(repo, arc_path, hashes, ext_dir) arc.close() # Extract all of the existing content. arc = ptf.PkgTarFile(name=arc_path, mode="r") arc.extractall(ext_dir) arc.close() # Now verify archive can still be used when index file # is omitted. os.unlink(arc_path) arc = ptf.PkgTarFile(name=arc_path, mode="w") for dirpath, dirnames, filenames in os.walk(ext_dir): list( map(add_entry, [f for f in filenames if f != "pkg5.index.0.gz"])) list(map(add_entry, dirnames)) arc.close() # Verify pkg(7) archive class extraction behaviour using # the new archive. arc = self.__verify_extract(repo, arc_path, hashes, ext_dir) arc.close() # Save an index for later. arc = pkg.p5p.Archive(arc_path, mode="r") saved_index = arc.get_index() arc.close() # Verify we can extract the archive reusing an index. arc = self.__verify_extract(repo, arc_path, hashes, ext_dir, archive_index=saved_index) arc.close() # Verify we throw an assert when opening a p5p in write mode. self.assertRaisesStringify(AssertionError, pkg.p5p.Archive, arc_path, mode="w", archive_index=saved_index) # Verify we can't extract archive members using a corrupted # index. arc = pkg.p5p.Archive(arc_path, mode="r", archive_index={"cats": 1234}) self.assertRaisesStringify(pkg.p5p.ArchiveErrors, arc.extract_catalog1, "catalog.attrs", ext_dir) self.assertRaisesStringify(pkg.p5p.ArchiveErrors, arc.extract_package_files, hashes, ext_dir) arc.close()
def test_02_add_package(self): """Verify that pkg(7) archive creation using add_package() works as expected. """ # Get repository. repo = self.get_repo(self.dc.get_repodir()) # Create a directory and copy package files from repository to # it (this is how pkgrecv stores content during republication # or when using --raw). dfroot = os.path.join(self.test_root, "pfiles") os.mkdir(dfroot, pkg.misc.PKG_DIR_MODE) foo_path = os.path.join(dfroot, "foo.p5m") portable.copyfile(repo.manifest(self.foo), foo_path) signed_path = os.path.join(dfroot, "signed.p5m") portable.copyfile(repo.manifest(self.signed), signed_path) quux_path = os.path.join(dfroot, "quux.p5m") portable.copyfile(repo.manifest(self.quux), quux_path) for rstore in repo.rstores: for dirpath, dirnames, filenames in os.walk(rstore.file_root): if not filenames: continue for f in filenames: portable.copyfile(os.path.join(dirpath, f), os.path.join(dfroot, f)) # Prep the archive. progtrack = pkg.client.progress.QuietProgressTracker() arc_path = os.path.join(self.test_root, "add_package.p5p") arc = pkg.p5p.Archive(arc_path, mode="w") # Create an archive with just one package. arc.add_package(self.foo, foo_path, dfroot) arc.close(progtrack=progtrack) # Verify the result. arc = ptf.PkgTarFile(name=arc_path, mode="r") expected = self.foo_expected actual = [m.name for m in arc.getmembers()] self.assertEqualDiff(expected, actual) # Prep a new archive. os.unlink(arc_path) arc = pkg.p5p.Archive(arc_path, mode="w") # Create an archive with multiple packages. # (Don't use progtrack this time.) arc.add_package(self.foo, foo_path, dfroot) arc.add_package(self.signed, signed_path, dfroot) arc.add_package(self.quux, quux_path, dfroot) arc.close() # Verify the result. arc = ptf.PkgTarFile(name=arc_path, mode="r") expected = self.multi_expected[:] action_certs = [ self.calc_pem_hash(t) for t in ( os.path.join(self.cs_dir, "cs1_ch5_ta1_cert.pem"), os.path.join(self.chain_certs_dir, "ch1_ta1_cert.pem"), os.path.join(self.chain_certs_dir, "ch2_ta1_cert.pem"), os.path.join(self.chain_certs_dir, "ch3_ta1_cert.pem"), os.path.join(self.chain_certs_dir, "ch4_ta1_cert.pem"), os.path.join(self.chain_certs_dir, "ch5_ta1_cert.pem"), os.path.join(self.chain_certs_dir, "ch1_ta3_cert.pem"), ) ] for hsh in action_certs: d = "publisher/test/file/{0}".format(hsh[0:2]) f = "{0}/{1}".format(d, hsh) expected.append(d) expected.append(f) actual = sorted(m.name for m in arc.getmembers()) self.assertEqualDiff(sorted(set(expected)), actual) os.unlink(arc_path) os.unlink(foo_path) os.unlink(quux_path) os.unlink(signed_path)
def test_01_add(self): """Verify that add() works as expected.""" # Prep the archive. arc_path = os.path.join(self.test_root, "add.p5p") arc = pkg.p5p.Archive(arc_path, mode="w") # add() permits addition of arbitrary files (intentionally); # it is also the routine that higher-level functions to add # package content use internally. Because of that, this # function does not strictly need standalone testing, but it # helps ensure all code paths for add() are tested. arc.add(self.test_root) tmp_root = os.path.join(self.test_root, "tmp") arc.add(tmp_root) for f in self.misc_files: src = os.path.join(self.test_root, f) # Ensure files are read-only mode so that file perm # normalization can be tested. os.chmod(src, pkg.misc.PKG_RO_FILE_MODE) arc.add(src) # Write out archive. arc.close() # Now open the archive and iterate through its contents and # verify that each member has the expected characteristics. arc = ptf.PkgTarFile(name=arc_path, mode="r") members = [m for m in arc.getmembers()] # Should be 11 files including package archive index and three # directories. actual = [m.name for m in members] self.assertEqual(len(actual), 11) expected = [ "pkg5.index.0.gz", "publisher", pkg.misc.relpath(self.test_root, "/"), pkg.misc.relpath(tmp_root, "/") ] expected.extend( pkg.misc.relpath(os.path.join(self.test_root, e), "/") for e in self.misc_files) expected.append("pkg5.repository") self.assertEqualDiff(expected, actual) for member in members: # All archive members should be a file or directory. self.assertTrue(member.isreg() or member.isdir()) if member.name == "pkg5.index.0.gz": assert member.isreg() comment = member.pax_headers.get("comment", "") self.assertEqual(comment, "pkg5.archive.version.0") continue if member.isdir(): # Verify directories were added with expected # mode. self.assertEqual(oct(member.mode), oct(pkg.misc.PKG_DIR_MODE)) elif member.isfile(): # Verify files were added with expected mode. self.assertEqual(oct(member.mode), oct(pkg.misc.PKG_FILE_MODE)) # Verify files and directories have expected ownership. self.assertEqual(member.uname, "root") self.assertEqual(member.gname, "root") self.assertEqual(member.uid, 0) self.assertEqual(member.gid, 0) os.unlink(arc_path)