def save_attrs(self, filenm="attrs"):
"""Save attributes from the in-memory catalog to a file
specified by filenm."""
tmpfile = None
assert not self.read_only
finalpath = os.path.normpath(
os.path.join(self.catalog_root, filenm))
try:
tmp_num, tmpfile = tempfile.mkstemp(
dir=self.catalog_root)
tfile = os.fdopen(tmp_num, "w")
for a in self.attrs.keys():
s = "S %s: %s\n" % (a, self.attrs[a])
tfile.write(s)
tfile.close()
os.chmod(tmpfile, self.file_mode)
portable.rename(tmpfile, finalpath)
except EnvironmentError, e:
# This may get called in a situation where
# the user does not have write access to the attrs
# file.
if tmpfile:
portable.remove(tmpfile)
if e.errno == errno.EACCES:
return
else:
raise
def __append_to_catalog(self, pkgstr):
"""Write string named pkgstr to the catalog. This
routine handles moving the catalog to a temporary file,
appending the new string, and renaming the temporary file
on top of the existing catalog."""
# Create tempfile
tmp_num, tmpfile = tempfile.mkstemp(dir=self.catalog_root)
try:
# use fdopen since we already have a filehandle
tfile = os.fdopen(tmp_num, "w")
except OSError:
portable.remove(tmpfile)
raise
# Try to open catalog file. If it doesn't exist,
# create an empty catalog file, and then open it read only.
try:
pfile = file(self.catalog_file, "rb")
except IOError, e:
if e.errno == errno.ENOENT:
# Creating an empty file
file(self.catalog_file, "wb").close()
pfile = file(self.catalog_file, "rb")
else:
portable.remove(tmpfile)
raise
def test_links(self):
self.pkgsend_bulk(self.rurl, self.link10)
self.image_create(self.rurl)
self.pkg("install link")
self.pkg_verify("link")
dpath = os.path.join(self.get_img_path(), "usr", "bin")
fpath = os.path.join(dpath, "bobcat")
lpath = os.path.join(dpath, "bobcat_link")
opath = os.path.join(dpath, "bobcat_owner")
# Change attributes of the symlink target
os.chown(fpath, 100, 2)
os.chmod(fpath, 0o664)
# Ownership of links themselves
os.lchown(lpath, 100, 2)
os.lchown(opath, 100, 2)
self.pkg_verify("-v -p /usr/bin/bobcat_link")
self.pkg_verify("-v -p /usr/bin/bobcat_owner")
# Remove the link
portable.remove(lpath)
self.pkg_verify("-v -p /usr/bin/bobcat_link", exit=1)
self.assertTrue("ERROR: missing: symbolic link" in self.output)
def test_clear_cache(self):
"""Verify that FactoredManifest.clear_cache() works as
expected."""
# Create FactoredManifest.
cache_dir = tempfile.mkdtemp(dir=self.test_root)
m1 = manifest.FactoredManifest("[email protected]",
cache_dir,
pathname=self.foo_content_p5m)
# Verify cache was created.
cfile_path = os.path.join(cache_dir, "manifest.dircache")
self.assertTrue(os.path.isfile(cfile_path))
# Create random file in cache_dir.
rfile_path = os.path.join(cache_dir, "junk")
self.make_file(rfile_path, "junk")
# Verify that clear_cache() removes all known files from
# cache_dir and will not remove directory if unknown are
# present.
m1.clear_cache(cache_dir)
for name in os.listdir(cache_dir):
self.assertEqualDiff(name, os.path.basename(rfile_path))
# Verify that clear_cache() removes cache_dir if empty.
portable.remove(rfile_path)
m1.clear_cache(cache_dir)
self.assertTrue(not os.path.exists(cache_dir))
def __append_to_catalog(self, pkgstr):
"""Write string named pkgstr to the catalog. This
routine handles moving the catalog to a temporary file,
appending the new string, and renaming the temporary file
on top of the existing catalog."""
# Create tempfile
tmp_num, tmpfile = tempfile.mkstemp(dir=self.catalog_root)
try:
# use fdopen since we already have a filehandle
tfile = os.fdopen(tmp_num, "w")
except OSError:
portable.remove(tmpfile)
raise
# Try to open catalog file. If it doesn't exist,
# create an empty catalog file, and then open it read only.
try:
pfile = file(self.catalog_file, "rb")
except IOError, e:
if e.errno == errno.ENOENT:
# Creating an empty file
file(self.catalog_file, "wb").close()
pfile = file(self.catalog_file, "rb")
else:
portable.remove(tmpfile)
raise
def test_verify_parsable_output(self):
"""Test parsable output."""
self.image_create(self.rurl)
self.pkg("install foo")
# Test invalid option combo.
self.pkg_verify("-v --parsable 0 foo", exit=2)
self.pkg_verify("-H --parsable 0 foo", exit=2)
# Test invalid option value.
self.pkg_verify("--parsable 1 foo", exit=2)
self.pkg_verify("--parsable 0 foo")
out_json = json.loads(self.output)
self.assertTrue("pkg://test/[email protected],5.11-0:20160229T095441Z" in
out_json["item-messages"])
fmri_entry = out_json["item-messages"][
"pkg://test/[email protected],5.11-0:20160229T095441Z"]
self.assertTrue(fmri_entry["messages"][0]["msg_level"] == "info")
self.assertTrue("file: usr/bin/bobcat" in fmri_entry)
# Verify should fail if file is missing.
fpath = os.path.join(self.get_img_path(), "usr", "bin", "bobcat")
portable.remove(fpath)
self.pkg_verify("--parsable 0 foo", exit=1)
out_json = json.loads(self.output)
self.assertTrue("pkg://test/[email protected],5.11-0:20160229T095441Z" in
out_json["item-messages"])
fmri_entry = out_json["item-messages"][
"pkg://test/[email protected],5.11-0:20160229T095441Z"]
self.assertTrue(fmri_entry["messages"][0]["msg_type"] == "general")
self.assertTrue(fmri_entry["messages"][0]["msg_level"] == "error")
self.assertTrue("file: usr/bin/bobcat" in fmri_entry)
self.assertTrue(
fmri_entry["file: usr/bin/bobcat"][0]["msg_type"] == "general")
self.assertTrue(
fmri_entry["file: usr/bin/bobcat"][0]["msg_level"] == "error")
def __append_to_log(self, logstr):
"""Write the string logstr into the proper update log.
This routine copies the existing contents of the log into
a temporary file, and then renames the new logfile into
place."""
# Get the path to the logfile, as well as its name
logpath, logfile = self._begin_log()
# Create a temporary file for new data
tmp_num, tmpfile = tempfile.mkstemp(dir=self.rootdir)
# Use fdopen since mkstemp gives us a filehandle
try:
tfile = os.fdopen(tmp_num, "w")
except OSError:
portable.remove(tmpfile)
raise
# Try to open logfile readonly. If it doesn't exist,
# create a new one, and then re-open it readonly.
try:
lfile = file(logpath, "rb")
except IOError, e:
if e.errno == errno.ENOENT:
# Creating an empty file
file(logpath, "wb").close()
lfile = file(logpath, "rb")
else:
portable.remove(tmpfile)
raise
def save_attrs(self, filenm="attrs"):
"""Save attributes from the in-memory catalog to a file
specified by filenm."""
tmpfile = None
assert not self.read_only
finalpath = os.path.normpath(os.path.join(self.catalog_root, filenm))
try:
tmp_num, tmpfile = tempfile.mkstemp(dir=self.catalog_root)
tfile = os.fdopen(tmp_num, "w")
for a in self.attrs.keys():
s = "S %s: %s\n" % (a, self.attrs[a])
tfile.write(s)
tfile.close()
os.chmod(tmpfile, self.file_mode)
portable.rename(tmpfile, finalpath)
except EnvironmentError, e:
# This may get called in a situation where
# the user does not have write access to the attrs
# file.
if tmpfile:
portable.remove(tmpfile)
if e.errno in (errno.EACCES, errno.EROFS):
return
else:
raise
def __append_to_log(self, logstr):
"""Write the string logstr into the proper update log.
This routine copies the existing contents of the log into
a temporary file, and then renames the new logfile into
place."""
# Get the path to the logfile, as well as its name
logpath, logfile = self._begin_log()
# Create a temporary file for new data
tmp_num, tmpfile = tempfile.mkstemp(dir=self.rootdir)
# Use fdopen since mkstemp gives us a filehandle
try:
tfile = os.fdopen(tmp_num, "w")
except OSError:
portable.remove(tmpfile)
raise
# Try to open logfile readonly. If it doesn't exist,
# create a new one, and then re-open it readonly.
try:
lfile = file(logpath, "rb")
except IOError, e:
if e.errno == errno.ENOENT:
# Creating an empty file
file(logpath, "wb").close()
lfile = file(logpath, "rb")
else:
portable.remove(tmpfile)
raise
def fetch_files_byhash(src_uri, cshashes, destdir, keep_compressed, tracker):
"""Given a list of tuples containing content hash, and size and download
the content from src_uri into destdir."""
def valid_file(h):
# XXX this should check data digest
fname = os.path.join(destdir, h)
if os.path.exists(fname):
if keep_compressed:
sz = cshashes[h][1]
else:
sz = cshashes[h][0]
if sz == 0:
return True
try:
fs = os.stat(fname)
except:
pass
else:
if fs.st_size == sz:
return True
return False
if src_uri.startswith("file://"):
try:
r = get_repo(src_uri)
except repo.RepositoryError, e:
abort(err=e)
for h in cshashes.keys():
dest = os.path.join(destdir, h)
# Check to see if the file already exists first, so the
# user can continue interrupted pkgrecv operations.
retrieve = not valid_file(h)
try:
if retrieve and keep_compressed:
src = r.file(h)
shutil.copy(src, dest)
elif retrieve:
src = file(r.file(h), "rb")
outfile = open(dest, "wb")
gunzip_from_stream(src, outfile)
outfile.close()
except (EnvironmentError,
repo.RepositoryError), e:
try:
portable.remove(dest)
except:
pass
abort(err=e)
tracker.download_add_progress(1, cshashes[h][2])
def destroy(root=None):
"""Removes the on-disk files for the catalog only."""
for fname in ("attrs", "catalog"):
path = os.path.normpath(os.path.join(root, fname))
try:
portable.remove(path)
except EnvironmentError, e:
if e.errno != errno.ENOENT:
raise
def destroy(root=None):
"""Removes the on-disk files for the catalog only."""
for fname in ("attrs", "catalog"):
path = os.path.normpath(os.path.join(root, fname))
try:
portable.remove(path)
except EnvironmentError, e:
if e.errno != errno.ENOENT:
raise
def fetch_files_byhash(src_uri, cshashes, destdir, keep_compressed, tracker):
"""Given a list of tuples containing content hash, and size and download
the content from src_uri into destdir."""
def valid_file(h):
# XXX this should check data digest
fname = os.path.join(destdir, h)
if os.path.exists(fname):
if keep_compressed:
sz = cshashes[h][1]
else:
sz = cshashes[h][0]
if sz == 0:
return True
try:
fs = os.stat(fname)
except:
pass
else:
if fs.st_size == sz:
return True
return False
if src_uri.startswith("file://"):
try:
r = get_repo(src_uri)
except repo.RepositoryError, e:
abort(err=e)
for h in cshashes.keys():
dest = os.path.join(destdir, h)
# Check to see if the file already exists first, so the
# user can continue interrupted pkgrecv operations.
retrieve = not valid_file(h)
try:
if retrieve and keep_compressed:
src = r.file(h)
shutil.copy(src, dest)
elif retrieve:
src = file(r.file(h), "rb")
outfile = open(dest, "wb")
gunzip_from_stream(src, outfile)
outfile.close()
except (EnvironmentError, repo.RepositoryError), e:
try:
portable.remove(dest)
except:
pass
abort(err=e)
tracker.download_add_progress(1, cshashes[h][2])
def __empty_repo(uri, arg_string):
if uri.startswith("http://"):
rurl = self.dcs[4].get_repo_url()
self.pkgrepo("remove -s %s '*'" % rurl)
# Refresh the depot to get it to realize that
# the catalog has changed.
self.dcs[4].refresh()
elif arg_string:
portable.remove(uri)
else:
self.pkgrepo("remove -s %s '*'" % uri)
def __append_to_catalog(self, pkgstr):
"""Write string named pkgstr to the catalog. This
routine handles moving the catalog to a temporary file,
appending the new string, and renaming the temporary file
on top of the existing catalog."""
# Create tempfile
tmp_num, tmpfile = tempfile.mkstemp(dir=self.catalog_root)
try:
# use fdopen since we already have a filehandle
tfile = os.fdopen(tmp_num, "w")
except OSError:
portable.remove(tmpfile)
raise
# Try to open catalog file. If it doesn't exist,
# create an empty catalog file, and then open it read only.
try:
pfile = open(self.catalog_file, "rb")
except IOError as e:
if e.errno == errno.ENOENT:
# Creating an empty file
open(self.catalog_file, "wb").close()
pfile = open(self.catalog_file, "rb")
else:
portable.remove(tmpfile)
raise
# Make sure we're at the start of the file
pfile.seek(0)
# Write all of the existing entries in the catalog
# into the tempfile. Then append the new lines at the
# end.
try:
for entry in pfile:
if entry == pkgstr:
raise CatalogException("Package {0} is already in "
"the catalog".format(pkgstr))
else:
tfile.write(entry)
tfile.write(pkgstr)
except Exception:
portable.remove(tmpfile)
raise
# Close our open files
pfile.close()
tfile.close()
# Set the permissions on the tempfile correctly.
# Mkstemp creates files as 600. Rename the new
# cataog on top of the old one.
try:
os.chmod(tmpfile, self.file_mode)
portable.rename(tmpfile, self.catalog_file)
except EnvironmentError:
portable.remove(tmpfile)
raise
def test_missing_manifest(self):
"""Check that missing manifests don't cause a traceback when
indexing or searching."""
rurl = self.dc.get_repo_url()
plist = self.pkgsend_bulk(rurl, self.example_pkg10)
api_obj = self.image_create(rurl)
self._api_install(api_obj, ["example_pkg"])
client_manifest_file = self.get_img_manifest_path(
fmri.PkgFmri(plist[0]))
portable.remove(client_manifest_file)
# Test search with a missing manifest.
self.pkg("search -l /bin", exit=1)
# Test rebuilding the index with a missing manifest.
self.pkg("rebuild-index")
def remove(self, pkgplan):
path = os.path.normpath(
os.path.sep.join((pkgplan.image.get_root(), self.attrs["path"])))
# are we supposed to save this file to restore it elsewhere
# or in another pkg?
if "save_file" in self.attrs:
self.save_file(pkgplan.image, path)
try:
# Make file writable so it can be deleted
os.chmod(path, stat.S_IWRITE | stat.S_IREAD)
portable.remove(path)
except OSError, e:
if e.errno != errno.ENOENT:
raise
def remove(self, pkgplan):
path = os.path.normpath(os.path.sep.join(
(pkgplan.image.get_root(), self.attrs["path"])))
# are we supposed to save this file to restore it elsewhere
# or in another pkg?
if "save_file" in self.attrs:
self.save_file(pkgplan.image, path)
try:
# Make file writable so it can be deleted
os.chmod(path, stat.S_IWRITE|stat.S_IREAD)
portable.remove(path)
except OSError,e:
if e.errno != errno.ENOENT:
raise
def __test_rec(duri, arg_string, pfmris):
self.debug("\n\nNow pkgrecv'ing to %s" % duri)
# It's necessary to use the -D option below because
# otherwise pkgrecv will fail because the manifest
# doesn't validate.
novalidate = "-D manifest_validate=Never "
# Check that invalid action attributes don't cause
# tracebacks.
self.pkgrecv(self.durl1, novalidate +
"-d %s %s %s" % (duri, arg_string,
" ".join(pfmris)), exit=pkgdefs.EXIT_OOPS)
for pfmri in pfmris:
__check_errout(pfmri)
self.assertEqual(__count_pulled_packages(duri), 0)
if arg_string:
portable.remove(duri)
self.pkgrecv(self.rurl1, novalidate +
"-d %s %s %s" % (duri, arg_string,
" ".join(pfmris)), exit=pkgdefs.EXIT_OOPS)
for pfmri in pfmris:
__check_errout(pfmri)
self.assertEqual(__count_pulled_packages(duri), 0)
if arg_string:
portable.remove(duri)
# Check that other packages are retrieved and the exit
# code reflects partial success.
self.pkgrecv(self.durl1, novalidate +
"-d %s %s -m all-timestamps '*'" %
(duri, arg_string), exit=pkgdefs.EXIT_PARTIAL)
for pfmri in pfmris:
__check_errout(pfmri)
self.assertEqual(__count_pulled_packages(duri),
len(self.published) - len(pfmris))
__empty_repo(duri, arg_string)
self.pkgrecv(self.rurl1, novalidate +
"-d %s %s -m all-timestamps '*'" %
(duri, arg_string), exit=pkgdefs.EXIT_PARTIAL)
for pfmri in pfmris:
__check_errout(pfmri)
self.assertEqual(__count_pulled_packages(duri),
len(self.published) - len(pfmris))
__empty_repo(duri, arg_string)
def testRemoveOfRunningExecutable(self):
if util.get_canonical_os_type() != 'windows':
return
import pkg.portable.os_windows as os_windows
cwd = os.getcwdu()
exefilesrc = 'C:\\Windows\\system32\\more.com'
self.assertTrue(os.path.exists(exefilesrc))
# create an image, copy an executable into it,
# run the executable, remove the executable
tdir1 = tempfile.mkdtemp()
img1 = image.Image(tdir1,
imgtype=image.IMG_USER,
should_exist=False,
user_provided_dir=True)
img1.history.client_name = "pkg-test"
img1.set_attrs(False,
"test",
origins=["http://localhost:10000"],
refresh_allowed=False)
exefile = os.path.join(tdir1, 'less.com')
shutil.copyfile(exefilesrc, exefile)
proc = subprocess.Popen([exefile], stdin=subprocess.PIPE)
self.assertRaises(OSError, os.unlink, exefile)
portable.remove(exefile)
self.assertTrue(not os.path.exists(exefile))
proc.communicate()
# Make sure that the moved executable gets deleted
# This is a white-box test
# To simulate running another process, we delete the cache
# and call get_trashdir as if another file was being moved
# to the trash.
os_windows.cached_image_info = []
os_windows.get_trashdir(exefile)
self.assertTrue(not os.path.exists(
os.path.join(img1.imgdir, os_windows.trashname)))
# cleanup
os.chdir(cwd)
shutil.rmtree(tdir1)
def __set_last_refreshed(self, value):
if not self.meta_root:
return
if value is not None and not isinstance(value, dt.datetime):
raise api_errors.BadRepositoryAttributeValue("last_refreshed",
value=value)
lcfile = os.path.join(self.meta_root, "last_refreshed")
if not value:
# If no value was provided, attempt to remove the
# tracking file.
try:
portable.remove(lcfile)
except EnvironmentError, e:
# If the file can't be removed due to
# permissions, a read-only filesystem, or
# because it doesn't exist, continue on.
if e.errno not in (errno.ENOENT, errno.EACCES, errno.EROFS):
raise
return
def remove(self, hashval):
"""This function removes the file associated with the name
"hashval"."""
if self.readonly:
raise NeedToModifyReadOnlyFileManager(hashval, "remove")
for l in self.layouts:
cur_path = l.lookup(hashval)
cur_full_path = os.path.join(self.root, cur_path)
try:
portable.remove(cur_full_path)
os.removedirs(os.path.dirname(cur_full_path))
except EnvironmentError as e:
if e.errno == errno.ENOENT or \
e.errno == errno.EEXIST:
pass
elif e.errno == errno.EACCES or \
e.errno == errno.EROFS:
raise FMPermissionsException(e.filename)
else:
raise
def __set_last_refreshed(self, value):
if not self.meta_root:
return
if value is not None and not isinstance(value, dt.datetime):
raise api_errors.BadRepositoryAttributeValue(
"last_refreshed", value=value)
lcfile = os.path.join(self.meta_root, "last_refreshed")
if not value:
# If no value was provided, attempt to remove the
# tracking file.
try:
portable.remove(lcfile)
except EnvironmentError, e:
# If the file can't be removed due to
# permissions, a read-only filesystem, or
# because it doesn't exist, continue on.
if e.errno not in (errno.ENOENT, errno.EACCES,
errno.EROFS):
raise
return
def remove(self, hashval):
"""This function removes the file associated with the name
"hashval"."""
if self.readonly:
raise NeedToModifyReadOnlyFileManager(hashval,
"remove")
for l in self.layouts:
cur_path = l.lookup(hashval)
cur_full_path = os.path.join(self.root, cur_path)
try:
portable.remove(cur_full_path)
os.removedirs(os.path.dirname(cur_full_path))
except EnvironmentError, e:
if e.errno == errno.ENOENT or \
e.errno == errno.EEXIST:
pass
elif e.errno == errno.EACCES or \
e.errno == errno.EROFS:
raise FMPermissionsException(e.filename)
else:
raise
def test_sysattrs(self):
"""Test that system attributes are verified correctly."""
if portable.osname != "sunos":
raise pkg5unittest.TestSkippedException(
"System attributes unsupported on this platform.")
self.pkgsend_bulk(self.rurl, [self.sysattr, self.sysattr2])
# Need to create an image in /var/tmp since sysattrs don't work
# in tmpfs.
old_img_path = self.img_path()
self.set_img_path(tempfile.mkdtemp(prefix="test-suite",
dir="/var/tmp"))
self.image_create(self.rurl)
self.pkg("install sysattr sysattr2")
self.pkg("verify")
fpath = os.path.join(self.img_path(), "p1/bobcat")
# Need to get creative here to remove the system attributes
# since you need the sys_linkdir privilege which we don't have:
# see run.py:393
# So we re-create the file with correct owner and mode and the
# only thing missing are the sysattrs.
portable.remove(fpath)
portable.copyfile(os.path.join(self.test_root, "bobcat"), fpath)
os.chmod(fpath, 0o555)
os.chown(fpath, -1, 2)
self.pkg("verify", exit=1)
for sattr in ('H', 'S'):
expected = "System attribute '{0}' not set".format(sattr)
self.assertTrue(expected in self.output,
"Missing in verify output: {0}".format(expected))
shutil.rmtree(self.img_path())
self.set_img_path(old_img_path)
def testRemoveOfRunningExecutable(self):
if util.get_canonical_os_type() != 'windows':
return
import pkg.portable.os_windows as os_windows
cwd = os.getcwdu()
exefilesrc = 'C:\\Windows\\system32\\more.com'
self.assert_(os.path.exists(exefilesrc))
# create an image, copy an executable into it,
# run the executable, remove the executable
tdir1 = tempfile.mkdtemp()
img1 = image.Image(tdir1, imgtype=image.IMG_USER,
should_exist=False, user_provided_dir=True)
img1.history.client_name = "pkg-test"
img1.set_attrs(False, "test",
origins=["http://localhost:10000"], refresh_allowed=False)
exefile = os.path.join(tdir1, 'less.com')
shutil.copyfile(exefilesrc, exefile)
proc = subprocess.Popen([exefile], stdin = subprocess.PIPE)
self.assertRaises(OSError, os.unlink, exefile)
portable.remove(exefile)
self.assert_(not os.path.exists(exefile))
proc.communicate()
# Make sure that the moved executable gets deleted
# This is a white-box test
# To simulate running another process, we delete the cache
# and call get_trashdir as if another file was being moved
# to the trash.
os_windows.cached_image_info = []
os_windows.get_trashdir(exefile)
self.assert_(not os.path.exists(os.path.join(img1.imgdir,
os_windows.trashname)))
# cleanup
os.chdir(cwd)
shutil.rmtree(tdir1)
def test_pkg_property_keyfiles(self):
"""key-files image property"""
def touch_file(p):
if not os.path.exists(os.path.dirname(p)):
os.makedirs(os.path.dirname(p))
fh = open(p, "w")
fh.write("")
fh.close()
self.pkgsend_bulk(self.rurl, [self.foo10, self.foo11])
vanilla = self.get_img_file_path("lib/.vanilla")
pecan = self.get_img_file_path("lib/.pecan")
self.image_create(self.rurl)
self.pkg("install [email protected]")
self.pkg("add-property-value key-files lib/.vanilla")
# Even adding a new keyfile property will fail as the
# image configuration cannot be loaded with a missing key-file.
self.pkg("add-property-value key-files lib/.pecan", exit=51)
touch_file(vanilla)
self.pkg("add-property-value key-files lib/.pecan")
touch_file(pecan)
self.pkg("property key-files")
self.assertTrue("vanilla" in self.output)
# pkg update should fail due to missing keyfile
portable.remove(vanilla)
self.pkg("update", exit=51)
self.assertTrue("Is everything mounted" in self.errout)
# and now succeed
touch_file(vanilla)
self.pkg("update")
# Make sure we're at the start of the file
pfile.seek(0)
# Write all of the existing entries in the catalog
# into the tempfile. Then append the new lines at the
# end.
try:
for entry in pfile:
if entry == pkgstr:
raise CatalogException("Package %s is already in "
"the catalog" % pkgstr)
else:
tfile.write(entry)
tfile.write(pkgstr)
except Exception:
portable.remove(tmpfile)
raise
# Close our open files
pfile.close()
tfile.close()
# Set the permissions on the tempfile correctly.
# Mkstemp creates files as 600. Rename the new
# cataog on top of the old one.
try:
os.chmod(tmpfile, self.file_mode)
portable.rename(tmpfile, self.catalog_file)
except EnvironmentError:
portable.remove(tmpfile)
raise
def remove_fsobj(self, pkgplan, path):
"""Shared logic for removing file and link objects."""
# Necessary since removal logic is reused by install.
fmri = pkgplan.destination_fmri
if not fmri:
fmri = pkgplan.origin_fmri
try:
portable.remove(path)
except EnvironmentError, e:
if e.errno == errno.ENOENT:
# Already gone; don't care.
return
elif e.errno == errno.EBUSY and os.path.ismount(path):
# User has replaced item with mountpoint, or a
# package has been poorly implemented.
err_txt = _("Unable to remove %s; it is in use "
"as a mountpoint. To continue, please "
"unmount the filesystem at the target "
"location and try again.") % path
raise apx.ActionExecutionError(self,
details=err_txt, error=e, fmri=fmri)
elif e.errno == errno.EBUSY:
# os.path.ismount() is broken for lofs
# filesystems, so give a more generic
# error.
err_txt = _("Unable to remove %s; it is in "
"use by the system, another process, or "
"as a mountpoint.") % path
raise apx.ActionExecutionError(self,
details=err_txt, error=e, fmri=fmri)
elif e.errno == errno.EPERM and \
not stat.S_ISDIR(os.lstat(path).st_mode):
# Was expecting a directory in this failure
# case, it is not, so raise the error.
raise
elif e.errno in (errno.EACCES, errno.EROFS):
# Raise these permissions exceptions as-is.
raise
elif e.errno != errno.EPERM:
# An unexpected error.
raise apx.ActionExecutionError(self, error=e,
fmri=fmri)
# Attempting to remove a directory as performed above
# gives EPERM. First, try to remove the directory,
# if it isn't empty, salvage it.
try:
os.rmdir(path)
except OSError, e:
if e.errno in (errno.EPERM, errno.EACCES):
# Raise permissions exceptions as-is.
raise
elif e.errno not in (errno.EEXIST,
errno.ENOTEMPTY):
# An unexpected error.
raise apx.ActionExecutionError(self,
error=e, fmri=fmri)
pkgplan.image.salvage(path)
def test_01_basics(self):
"""Ensure that verify returns failure as expected when packages
are not correctly installed."""
# XXX either this should be more comprehensive or more testing
# needs to be added somewhere else appropriate.
self.image_create(self.rurl)
# Create a dummy publisher so that test publisher can be removed
# and added back as needed.
self.pkg("set-publisher -P ignored")
# Should fail since foo is not installed.
self.pkg("verify foo", exit=1)
# Now install package.
self.pkg("install foo")
# Should not fail since informational messages are not
# fatal.
self.pkg("verify foo")
# Should not fail if publisher is disabled and package is ok.
self.pkg("set-publisher -d test")
self.pkg("verify foo")
# Should not fail if publisher is removed and package is ok.
self.pkg("unset-publisher test")
self.pkg("verify foo")
# Should fail with exit code 1 if publisher is removed and
# package is not ok.
portable.remove(os.path.join(self.get_img_path(), "usr", "bin",
"bobcat"))
self.pkg("verify foo", exit=1)
self.pkg("set-publisher -p %s" % self.rurl)
self.pkg("fix foo")
# Informational messages should not be output unless -v
# is provided.
self.pkg("set-publisher -p %s" % self.rurl)
self.pkg("verify foo | grep bobcat", exit=1)
self.pkg("verify -v foo | grep bobcat")
# Verify shouldn't care if timestamp has changed on
# preserved files.
fpath = os.path.join(self.get_img_path(), "etc", "preserved")
ctime = time.time() - 1240
os.utime(fpath, (ctime, ctime))
self.pkg("verify foo")
# Verify should fail if file is missing.
fpath = os.path.join(self.get_img_path(), "usr", "bin",
"bobcat")
portable.remove(fpath)
self.pkg("verify foo", exit=1)
# Now verify that verify warnings are not fatal (because
# package contained bobcat!).
self.pkg("fix")
fpath = os.path.join(self.get_img_path(), "etc",
"driver_aliases")
with open(fpath, "ab+") as f:
out = ""
for l in f:
if l.find("zigit") != -1:
nl = l.replace("1234", "4321")
out += nl
out += l
f.truncate(0)
f.write(out)
# Verify should find the extra alias...
self.pkg("verify -v foo | grep 4321")
# ...but it should not be treated as a fatal error.
self.pkg("verify foo")
class UpdateLog(object):
"""The update log is a mechanism that allows clients and servers to
make incremental updates to their package catalogs. The server logs
whether it has added or removed a package, the time when the action
occurred, and the name of the package added or removed. The client
requests a list of actions that have been applied to the server's
catalog since a particular time in the past. The server is then able
to send this list of actions, allowing the client to apply these
changes to its catalog.
This allows the client to obtain incremental updates to its catalog,
instead of having to download an entire (and largely duplicated)
catalog each time a refresh is requested.
The UpdateLog must have an associated catalog; however,
Catalogs are not required to have an UpdateLog. The UpdateLog
allows catalogs to support incremental updates.
The catalog format is a + or -, an isoformat timestamp, and a catalog
entry in server-side format. They must be in order and separated by
spaces."""
def __init__(self, update_root, cat, maxfiles=336):
"""Create an instance of the UpdateLog. "update_root" is
the root directory for the update log files.
maxfiles is the maximum number of logfiles that
the UpdateLog will keep. A new file is added
for each hour in which there is an update. The
default value of 336 means that we keep 336 hours, or
14 days worth of log history."""
self.rootdir = update_root
self.maxfiles = maxfiles
self.catalog = cat
self.first_update = None
self.last_update = None
self.curfiles = 0
self.logfiles = []
self.logfile_size = {}
self.updatelog_lock = threading.Lock()
if not os.path.exists(update_root):
os.makedirs(update_root)
self._setup_logfiles()
def add_package(self, pfmri, critical=False):
"""Record that the catalog has added "pfmri"."""
self.updatelog_lock.acquire()
try:
# First add FMRI to catalog
ts = self.catalog.add_fmri(pfmri, critical)
# Now add update to updatelog
self._check_logs()
if critical:
entry_type = "C"
else:
entry_type = "V"
# The format for catalog C and V records is described
# in the docstrings for the Catalog class.
logstr = "+ %s %s %s\n" % \
(ts.isoformat(), entry_type,
pfmri.get_fmri(anarchy=True))
self.__append_to_log(logstr)
self.last_update = ts
finally:
self.updatelog_lock.release()
return ts
def _begin_log(self):
"""Return the path to a logfile. If we haven't written
any updates yet, do some additional bookkeeping."""
filenm = time.strftime("%Y%m%d%H")
ftime = datetime.datetime(*time.strptime(filenm, "%Y%m%d%H")[0:6])
delta = datetime.timedelta(hours=1)
path = os.path.join(self.rootdir, filenm)
if filenm not in self.logfiles:
self.logfiles.append(filenm)
self.curfiles += 1
if not self.first_update:
self.first_update = ftime
return path, filenm
def _check_logs(self):
"""Check to see if maximum number of logfiles has been
exceeded. If so, rotate the logs. Also, if a log is
open, check to see if it needs to be closed."""
if self.curfiles < self.maxfiles:
return
excess = self.curfiles - self.maxfiles
to_remove = self.logfiles[0:excess]
for r in to_remove:
filepath = os.path.join(self.rootdir, "%s" % r)
os.unlink(filepath)
self.curfiles -= 1
if r in self.logfile_size:
del self.logfile_size[r]
del self.logfiles[0:excess]
self.first_update = datetime.datetime(
*time.strptime(self.logfiles[0], "%Y%m%d%H")[0:6])
def __append_to_log(self, logstr):
"""Write the string logstr into the proper update log.
This routine copies the existing contents of the log into
a temporary file, and then renames the new logfile into
place."""
# Get the path to the logfile, as well as its name
logpath, logfile = self._begin_log()
# Create a temporary file for new data
tmp_num, tmpfile = tempfile.mkstemp(dir=self.rootdir)
# Use fdopen since mkstemp gives us a filehandle
try:
tfile = os.fdopen(tmp_num, "w")
except OSError:
portable.remove(tmpfile)
raise
# Try to open logfile readonly. If it doesn't exist,
# create a new one, and then re-open it readonly.
try:
lfile = file(logpath, "rb")
except IOError, e:
if e.errno == errno.ENOENT:
# Creating an empty file
file(logpath, "wb").close()
lfile = file(logpath, "rb")
else:
portable.remove(tmpfile)
raise
# Make sure we're at the start of the file
lfile.seek(0)
# Write existing lines in old file into new file.
# Then append the new line.
try:
for entry in lfile:
tfile.write(entry)
tfile.write(logstr)
except Exception:
portable.remove(tmpfile)
raise
# If this routine is updating a logfile that already
# has size information, replace the size information
# with the size of the new file. Use tell to grab the
# offset at the end of the file, instead of stat.
# (At least in this case)
if logfile in self.logfile_size:
self.logfile_size[logfile] = tfile.tell()
lfile.close()
tfile.close()
# Change the permissions on the tempfile, since
# mkstemp uses mode 600. Rename the tempfile into
# place as the new logfile.
try:
os.chmod(tmpfile, catalog.ServerCatalog.file_mode)
portable.rename(tmpfile, logpath)
except EnvironmentError:
portable.remove(tmpfile)
raise
def test_01_basics(self):
"""Ensure that verify returns failure as expected when packages
are not correctly installed."""
# XXX either this should be more comprehensive or more testing
# needs to be added somewhere else appropriate.
self.image_create(self.rurl)
# Create a dummy publisher so that test publisher can be removed
# and added back as needed.
self.pkg("set-publisher -P ignored")
# Should fail since foo is not installed.
self.pkg_verify("foo", exit=1)
self.assert_("Unexpected Exception" not in self.output)
# Now install package.
self.pkg("install foo")
# Should not fail since informational messages are not
# fatal.
self.pkg_verify("foo")
# Unprivileged users don't cause a traceback.
retcode, output = self.pkg_verify("foo",
su_wrap=True,
out=True,
exit=1)
self.assert_("Traceback" not in output)
# Should not output anything when using -q.
self.pkg_verify("-q foo")
assert (self.output == "")
# Should not fail if publisher is disabled and package is ok.
self.pkg("set-publisher -d test")
self.pkg_verify("foo")
# Should not fail if publisher is removed and package is ok.
self.pkg("unset-publisher test")
self.pkg_verify("foo")
# Should fail with exit code 1 if publisher is removed and
# package is not ok.
portable.remove(
os.path.join(self.get_img_path(), "usr", "bin", "bobcat"))
self.pkg_verify("foo", exit=1)
self.assert_("Unexpected Exception" not in self.output)
self.assert_("PACKAGE" in self.output and "STATUS" in self.output)
# Test that "-H" works as expected.
self.pkg_verify("foo -H", exit=1)
self.assert_("PACKAGE" not in self.output
and "STATUS" not in self.output)
# Should not output anything when using -q.
self.pkg_verify("-q foo", exit=1)
assert (self.output == "")
self.pkg("set-publisher -p {0}".format(self.rurl))
self.pkg("fix foo")
# Informational messages should not be output unless -v
# is provided.
self.pkg("set-publisher -p {0}".format(self.rurl))
self.pkg_verify("foo | grep bobcat", exit=1)
self.pkg_verify("-v foo | grep bobcat")
# Verify shouldn't care if timestamp has changed on
# preserved files.
fpath = os.path.join(self.get_img_path(), "etc", "preserved")
ctime = time.time() - 1240
os.utime(fpath, (ctime, ctime))
self.pkg_verify("foo")
# Verify should fail if file is missing.
fpath = os.path.join(self.get_img_path(), "usr", "bin", "bobcat")
portable.remove(fpath)
self.pkg_verify("foo", exit=1)
# Now verify that verify warnings are not fatal (because
# package contained bobcat!).
self.pkg("fix")
fpath = os.path.join(self.get_img_path(), "etc", "driver_aliases")
with open(fpath, "ab+") as f:
out = ""
for l in f:
if l.find("zigit") != -1:
nl = l.replace("1234", "4321")
out += nl
out += l
f.truncate(0)
f.write(out)
# Verify should find the extra alias...
self.pkg_verify("-v foo | grep 4321")
# ...but it should not be treated as a fatal error.
self.pkg_verify("foo")
def test_fix_overlay(self):
"""Test that pkg verify / fix should tell the users to look at
the overlaying package in the error message if fix won't repair
the overlaid package."""
file_path = "etc/gss/mech"
file_path_1 = "etc/gss/mech_1"
self.image_create(self.rurl)
pfmri_gss = self.plist["[email protected]"]
pfmri_krb = self.plist["[email protected]"]
pfmri_sysattr = self.plist["[email protected]"]
pfmri_sysattr_o = self.plist["[email protected]"]
# First, only install the package that has a file with
# attribute overlay=allow.
self.pkg("install gss")
self.file_exists(file_path)
self.file_remove(file_path)
self.file_doesnt_exist(file_path)
# Verify should report an error if the file is missing.
self.pkg("verify -v gss", exit=1)
# Fix should be able to repair the file.
self.pkg("fix -v gss")
self.file_exists(file_path)
self.__do_alter_verify(pfmri_gss)
# Install the overlaying package.
self.pkg("install krb5")
self.file_exists(file_path)
self.file_remove(file_path)
self.file_doesnt_exist(file_path)
# Now pkg verify should still report an error on the overlaid
# package and tell the users to verify the overlaying package.
self.pkg("verify gss", exit=1)
self.assertTrue("package: {0}".format(
pfmri_krb.get_pkg_stem(anarchy=True)) in self.output)
# Verify should report an error on the overlaying package.
self.pkg("verify krb5", exit=1)
# Fix won't repair the overlaid package but will tell the users
# to fix the overlaying package in the verbose mode.
self.pkg("fix gss", exit=4)
self.pkg("fix -v gss", exit=4)
self.assertTrue("Could not repair: {0}".format(pfmri_gss) in
self.output)
self.assertTrue("package: {0}".format(
pfmri_krb.get_pkg_stem(anarchy=True)) in self.output)
self.file_doesnt_exist(file_path)
# Fix should be able to repair the file by fixing the overlaying
# package.
self.pkg("fix -v pkg:/krb5")
self.pkg("verify gss")
self.file_exists(file_path)
# Test that multiple overlaid files are missing.
self.file_remove(file_path)
self.file_remove(file_path_1)
self.pkg("verify gss", exit=1)
# Test that the overlay warning only emits once for each
# package.
self.pkg("verify gss | grep 'verify or fix' | wc -l | grep 1")
self.pkg("fix krb5")
# Test the owner, group and mode change.
self.__do_alter_verify(pfmri_gss, verbose=True, exit=4)
self.assertTrue("Could not repair: {0}".format(pfmri_gss) in
self.output)
self.assertTrue("package: {0}".format(
pfmri_krb.get_pkg_stem(anarchy=True)) in self.output)
self.__do_alter_verify(pfmri_krb, verbose=True)
# Test that verify / fix on system wide could report / fix the
# error on the overlaid and overlaying packges.
self.file_remove(file_path)
self.pkg("verify", exit=1)
# Test that verify / fix on all packages should not emit the
# overlaying warning.
self.assertTrue("verify or fix" not in self.output)
self.pkg("fix")
self.assertTrue("verify or fix" not in self.output)
self.pkg("verify")
self.file_exists(file_path)
# Test different file types install. Since fix will repair the
# overlaid package in this case, we don't need to tell the users
# to look at the overlaying package.
self.pkg("-D broken-conflicting-action-handling=1 install "
"dupfile duplink")
self.pkg("verify dupfile", exit=1)
self.pkg("fix dupfile")
self.pkg("verify dupfile")
# Test overlaid package that contains system attribute error.
self.set_img_path(tempfile.mkdtemp(prefix="test-suite",
dir="/var/tmp"))
self.image_create(self.rurl)
self.pkg("install sysattr")
fpath = os.path.join(self.img_path(), "amber1")
# Install the overlaying package.
self.pkg("install sysattr_overlay")
portable.remove(fpath)
portable.copyfile(os.path.join(self.test_root, "amber1"),
fpath)
os.chmod(fpath, 0o555)
os.chown(fpath, -1, 2)
self.pkg("verify sysattr", exit=1)
self.pkg("fix -v sysattr", exit=4)
self.assertTrue("Could not repair: {0}".format(pfmri_sysattr) in
self.output, self.plist)
self.assertTrue("package: {0}".format(
pfmri_sysattr_o.get_pkg_stem(anarchy=True)) in self.output)
self.pkg("fix sysattr_overlay")
self.pkg("verify sysattr")
self.image_destroy()
def test_01_basics(self):
"""Ensure that verify returns failure as expected when packages
are not correctly installed."""
# XXX either this should be more comprehensive or more testing
# needs to be added somewhere else appropriate.
self.image_create(self.rurl)
# Create a dummy publisher so that test publisher can be removed
# and added back as needed.
self.pkg("set-publisher -P ignored")
# Should fail since foo is not installed.
self.pkg_verify("foo", exit=1)
self.assertTrue("Unexpected Exception" not in self.output)
# Now install package.
self.pkg("install foo")
# Should not fail since informational messages are not
# fatal.
self.pkg_verify("foo")
# Unprivileged users don't cause a traceback.
retcode, output = self.pkg_verify("foo",
su_wrap=True,
out=True,
exit=1)
self.assertTrue("Traceback" not in output)
# Should not output anything when using -q.
self.pkg_verify("-q foo")
assert (self.output == "")
# Should not fail since the path exists in the package
# and is intact.
self.pkg_verify("-v -p /etc/name_to_major")
self.assertTrue("foo" in self.output
and "etc/name_to_major" not in self.output)
self.pkg_verify("-v -p /usr/bin/bobcat_link")
self.assertTrue("OK" in self.output)
self.pkg_verify("-v -p /usr")
self.assertTrue(self.output.count("OK") == 1)
# Should output path not found.
self.pkg_verify("-p nonexist")
self.assertTrue("not found" in self.output)
# Should not fail if publisher is disabled and package is ok.
self.pkg("set-publisher -d test")
self.pkg_verify("foo")
# Should not fail if publisher is removed and package is ok.
self.pkg("unset-publisher test")
self.pkg_verify("foo")
# Should fail with exit code 1 if publisher is removed and
# package is not ok.
portable.remove(
os.path.join(self.get_img_path(), "usr", "bin", "bobcat"))
self.pkg_verify("foo", exit=1)
self.assertTrue("Unexpected Exception" not in self.output)
self.assertTrue("PACKAGE" in self.output and "STATUS" in self.output)
# Should fail with exit code 2 because of invalid option combo.
self.pkg_verify("-p /usr/bin/bobcat --unpackaged", exit=2)
self.pkg_verify("-p /usr/bin/bobcat --unpackaged-only", exit=2)
# Should fail with exit code 1 because the file is removed
# and the package is not ok.
self.pkg_verify("-p /usr/bin/bobcat", exit=1)
self.assertTrue("PACKAGE" in self.output
and self.output.count("ERROR") == 2)
self.assertTrue("usr/bin/bobcat" in self.output)
# Test that "-H" works as expected.
self.pkg_verify("foo -H", exit=1)
self.assertTrue("PACKAGE" not in self.output
and "STATUS" not in self.output)
# Should not output anything when using -q.
self.pkg_verify("-q foo", exit=1)
assert (self.output == "")
self.pkg("set-publisher -p {0}".format(self.rurl))
self.pkg("fix foo")
# Informational messages should not be output unless -v
# is provided.
self.pkg("set-publisher -p {0}".format(self.rurl))
self.pkg_verify("foo | grep bobcat", exit=1)
self.pkg_verify("-v foo | grep bobcat")
# Verify shouldn't care if timestamp has changed on
# preserved files.
fpath = os.path.join(self.get_img_path(), "etc", "preserved")
ctime = time.time() - 1240
os.utime(fpath, (ctime, ctime))
self.pkg_verify("foo")
# Verify should fail if file is missing.
fpath = os.path.join(self.get_img_path(), "usr", "bin", "bobcat")
portable.remove(fpath)
self.pkg_verify("foo", exit=1)
# Now verify that verify warnings are not fatal (because
# package contained bobcat!).
self.pkg("fix")
fpath = os.path.join(self.get_img_path(), "etc", "driver_aliases")
with open(fpath, "r+") as f:
out = ""
for l in f:
if l.find("zigit") != -1:
nl = l.replace("1234", "4321")
out += nl
out += l
f.seek(0)
f.write(out)
# Verify should find the extra alias and it should be treated
# as a warning.
self.pkg_verify("-v foo")
#self.assertTrue("4321" in self.output)
self.assertTrue("4321" in self.output and "WARNING" in self.output)
# Test that warnings are displayed by default.
self.pkg_verify("foo")
self.assertTrue("4321" in self.output and "WARNING" in self.output)
# Verify on system wide should also find the extra alias.
self.pkg_verify("")
self.assertTrue("4321" in self.output and "WARNING" in self.output)
def test_fix_overlay(self):
"""Test that pkg verify / fix should tell the users to look at
the overlaying package in the error message if fix won't repair
the overlaid package."""
file_path = "etc/gss/mech"
file_path_1 = "etc/gss/mech_1"
self.image_create(self.rurl)
pfmri_gss = self.plist["[email protected]"]
pfmri_krb = self.plist["[email protected]"]
pfmri_sysattr = self.plist["[email protected]"]
pfmri_sysattr_o = self.plist["[email protected]"]
# First, only install the package that has a file with
# attribute overlay=allow.
self.pkg("install gss")
# Path verification should report ok.
self.pkg("verify -v -p {0}".format(file_path))
self.assertTrue("OK" in self.output and file_path not in self.output
and pfmri_gss.get_pkg_stem() in self.output)
self.file_exists(file_path)
self.file_remove(file_path)
self.file_doesnt_exist(file_path)
# Verify should report an error if the file is missing.
self.pkg("verify -v gss", exit=1)
# Path verification should report error.
self.pkg("verify -v -p {0}".format(file_path), exit=1)
self.assertTrue("OK" not in self.output and "ERROR" in self.output)
self.assertTrue(file_path in self.output and \
pfmri_gss.get_pkg_stem() in self.output)
# Fix should be able to repair the file.
self.pkg("fix -v gss")
self.file_exists(file_path)
# On-disk action attributes should be changed and should be
# fixed.
self.__do_alter_verify(pfmri_gss, exit=0)
# Install the overlaying package.
self.pkg("install krb5")
# Path verification should report ok for both the overlaid package
# and the overlaying package.
self.pkg("verify -v -p {0}".format(file_path))
self.assertTrue(
self.output.count("OK") == 2 and "ERROR" not in self.output)
self.assertTrue(pfmri_krb.get_pkg_stem() in self.output
and pfmri_gss.get_pkg_stem() in self.output)
self.pkg("verify -v -p {0} gss".format(file_path))
self.assertTrue(
self.output.count("OK") == 2 and "ERROR" not in self.output)
self.assertTrue(pfmri_krb.get_pkg_stem() in self.output
and pfmri_gss.get_pkg_stem() in self.output)
self.file_exists(file_path)
self.file_remove(file_path)
self.file_doesnt_exist(file_path)
# Path verification should report error for both the overlaid package
# and the overlaying package.
self.pkg("verify -v -p {0}".format(file_path), exit=1)
self.assertTrue("OK" not in self.output
and self.output.count("ERROR") == 4)
self.assertTrue(pfmri_krb.get_pkg_stem() in self.output
and pfmri_gss.get_pkg_stem() in self.output)
self.pkg("verify -v -p {0} gss".format(file_path), exit=1)
self.assertTrue("OK" not in self.output
and self.output.count("ERROR") == 4)
self.assertTrue(pfmri_krb.get_pkg_stem() in self.output
and pfmri_gss.get_pkg_stem() in self.output)
# Now pkg verify should still report an error on the overlaid
# package and tell the users it is from the overlaying package.
self.pkg("verify gss", exit=1)
self.assertTrue("from {0}".format(pfmri_krb.get_pkg_stem(
anarchy=True)) in self.output)
# Verify should report an error on the overlaying package.
self.pkg("verify krb5", exit=1)
# Fix will fix the overlaying action (from krb5) for gss
# directly.
self.pkg("fix gss")
self.file_exists(file_path)
# Fix has fixed that one before.
self.pkg("fix -v pkg:/krb5", exit=4)
self.pkg("verify gss")
self.file_exists(file_path)
# Test that multiple overlaid files are missing.
self.file_remove(file_path)
self.file_remove(file_path_1)
self.pkg("verify gss", exit=1)
self.pkg("fix krb5")
# Test the owner, group and mode change.
self.__do_alter_verify(pfmri_gss, verbose=True)
self.__do_alter_verify(pfmri_krb, verbose=True)
# Test that verify / fix on system wide could report / fix the
# error on the overlaid and overlaying packges.
self.file_remove(file_path)
self.pkg("verify", exit=1)
self.assertTrue("(from " in self.output)
self.pkg("fix")
self.assertTrue("(from " in self.output)
self.pkg("verify")
self.file_exists(file_path)
# Test different file types install. Since fix will repair the
# overlaid package in this case, we don't need to tell the users
# to look at the overlaying package.
self.pkg("-D broken-conflicting-action-handling=1 install "
"dupfile duplink")
self.pkg("verify dupfile", exit=1)
self.pkg("fix dupfile")
self.pkg("verify dupfile")
# Test overlaid package that contains system attribute error.
self.set_img_path(tempfile.mkdtemp(prefix="test-suite",
dir="/var/tmp"))
self.image_create(self.rurl)
self.pkg("install sysattr")
fpath = os.path.join(self.img_path(), "amber1")
# Install the overlaying package.
self.pkg("install sysattr_overlay")
portable.remove(fpath)
portable.copyfile(os.path.join(self.test_root, "amber1"), fpath)
os.chmod(fpath, 0o555)
os.chown(fpath, -1, 2)
self.pkg("verify sysattr", exit=1)
self.pkg("fix -v sysattr")
self.assertTrue(
"from {0}".format(pfmri_sysattr_o.get_pkg_stem(
anarchy=True)) in self.output)
self.pkg("fix sysattr_overlay", exit=4)
self.pkg("verify sysattr")
self.image_destroy()
def __reset_file(self):
"""Remove and recreate test file to clear sys attrs."""
portable.remove(self.test_fn)
self.test_fh, self.test_fn = tempfile.mkstemp(
dir=self.test_path)
def tearDown(self):
portable.remove(self.test_fn)
os.rmdir(self.test_path)
def remove_fsobj(self, pkgplan, path):
"""Shared logic for removing file and link objects."""
# Necessary since removal logic is reused by install.
fmri = pkgplan.destination_fmri
if not fmri:
fmri = pkgplan.origin_fmri
try:
portable.remove(path)
except EnvironmentError, e:
if e.errno == errno.ENOENT:
# Already gone; don't care.
return
elif e.errno == errno.EBUSY and os.path.ismount(path):
# User has replaced item with mountpoint, or a
# package has been poorly implemented.
err_txt = _("Unable to remove %s; it is in use "
"as a mountpoint. To continue, please "
"unmount the filesystem at the target "
"location and try again.") % path
raise apx.ActionExecutionError(self,
details=err_txt, error=e, fmri=fmri)
elif e.errno == errno.EBUSY:
# os.path.ismount() is broken for lofs
# filesystems, so give a more generic
# error.
err_txt = _("Unable to remove %s; it is in "
"use by the system, another process, or "
"as a mountpoint.") % path
raise apx.ActionExecutionError(self,
details=err_txt, error=e, fmri=fmri)
elif e.errno == errno.EPERM and \
not stat.S_ISDIR(os.lstat(path).st_mode):
# Was expecting a directory in this failure
# case, it is not, so raise the error.
raise
elif e.errno in (errno.EACCES, errno.EROFS):
# Raise these permissions exceptions as-is.
raise
elif e.errno != errno.EPERM:
# An unexpected error.
raise apx.ActionExecutionError(self, error=e,
fmri=fmri)
# Attempting to remove a directory as performed above
# gives EPERM. First, try to remove the directory,
# if it isn't empty, salvage it.
try:
os.rmdir(path)
except OSError, e:
if e.errno in (errno.EPERM, errno.EACCES):
# Raise permissions exceptions as-is.
raise
elif e.errno not in (errno.EEXIST,
errno.ENOTEMPTY):
# An unexpected error.
raise apx.ActionExecutionError(self,
error=e, fmri=fmri)
pkgplan.image.salvage(path)
info.name)
outfile = open(fpath, "wb")
gunzip_from_stream(gzfobj, outfile)
outfile.close()
gzfobj.close()
else:
# We want to keep the files compressed
# on disk.
tar_stream.extract_to(info, tmpdir,
info.name)
# Copy the file into place (rename can cause a cross-
# link device failure) and then remove the original.
src = os.path.join(tmpdir, info.name)
shutil.copy(src, os.path.join(destdir, info.name))
portable.remove(src)
tracker.download_add_progress(1, cshashes[info.name][2])
except KeyboardInterrupt:
raise
except:
abort(err=_("Unable to extract file: %s") % info.name)
shutil.rmtree(tmpdirs.pop(), True)
tar_stream.close()
f.close()
def list_newest_fmris(fmri_list):
"""List the provided fmris."""
def write(self, path):
"""Write the configuration to the given directory"""
cp = ConfigParser.SafeConfigParser()
# XXX the use of the disabled_auth file can be removed when
# compatibility with the older code is no longer needed
da = ConfigParser.SafeConfigParser()
# For compatibility, the preferred-publisher is written out
# as the preferred-authority. Modify a copy so that we don't
# change the in-memory copy.
props = self.properties.copy()
try:
del props["preferred-publisher"]
except KeyError:
pass
props["preferred-authority"] = self.preferred_publisher
cp.add_section("property")
for p in props:
cp.set("property", p,
props[p].encode("utf-8"))
cp.add_section("filter")
for f in self.filters:
cp.set("filter", f, str(self.filters[f]))
cp.add_section("variant")
for f in self.variants:
cp.set("variant", f, str(self.variants[f]))
for prefix in self.publishers:
pub = self.publishers[prefix]
section = "authority_%s" % pub.prefix
c = cp
if pub.disabled:
c = da
c.add_section(section)
c.set(section, "alias", str(pub.alias))
c.set(section, "prefix", str(pub.prefix))
c.set(section, "disabled", str(pub.disabled))
repo = pub.selected_repository
c.set(section, "origin", repo.origins[0].uri)
c.set(section, "mirrors",
str([u.uri for u in repo.mirrors]))
#
# For zones, where the reachability of an absolute path
# changes depending on whether you're in the zone or
# not. So we have a different policy: ssl_key and
# ssl_cert are treated as zone root relative.
#
ngz = self.variants.get("variant.opensolaris.zone",
"global") == "nonglobal"
p = str(pub["ssl_key"])
if ngz and self.__imgroot != os.sep and p != "None":
# Trim the imageroot from the path.
if p.startswith(self.__imgroot):
p = p[len(self.__imgroot):]
# XXX this should be per origin or mirror
c.set(section, "ssl_key", p)
p = str(pub["ssl_cert"])
if ngz and self.__imgroot != os.sep and p != "None":
if p.startswith(self.__imgroot):
p = p[len(self.__imgroot):]
# XXX this should be per origin or mirror
c.set(section, "ssl_cert", p)
# XXX this should really be client_uuid, but is being
# left with this name for compatibility with older
# clients.
c.set(section, "uuid", str(pub.client_uuid))
# Write selected repository data.
# XXX this is temporary until a switch to a more
# expressive configuration format is made.
repo = pub.selected_repository
repo_data = {
"collection_type": repo.collection_type,
"description": repo.description,
"legal_uris": [u.uri for u in repo.legal_uris],
"name": repo.name,
"refresh_seconds": repo.refresh_seconds,
"registered": repo.registered,
"registration_uri": repo.registration_uri,
"related_uris": [u.uri for u in repo.related_uris],
"sort_policy": repo.sort_policy,
}
for key, val in repo_data.iteritems():
c.set(section, "repo.%s" % key, str(val))
# XXX Child images
for afile, acp in [(CFG_FILE, cp), (DA_FILE, da)]:
thefile = os.path.join(path, afile)
if len(acp.sections()) == 0:
if os.path.exists(thefile):
portable.remove(thefile)
continue
try:
f = open(thefile, "w")
except EnvironmentError, e:
if e.errno == errno.EACCES:
raise api_errors.PermissionsException(
e.filename)
raise
acp.write(f)
def __test_offline_fix(self, configure_cb, offline_cb, online_cb):
"""Private helper function for ensuring that offline operation
is supported for 'pkg fix' when no package data retrieval is
required."""
# If only attributes are wrong and no local modification
# is on the file content, fix doesn't need to download the
# file data.
# Test the system attribute.
# Need to create an image in /var/tmp since sysattrs don't work
# in tmpfs.
old_img_path = self.img_path()
self.set_img_path(tempfile.mkdtemp(prefix="test-suite",
dir="/var/tmp"))
self.image_create(self.durl)
configure_cb()
self.pkg("install sysattr")
self.pkg("verify")
fpath = os.path.join(self.img_path(), "amber1")
# Need to get creative here to remove the system attributes
# since you need the sys_linkdir privilege which we don't have:
# see run.py:393
# So we re-create the file with correct owner and mode and the
# only thing missing are the sysattrs.
portable.remove(fpath)
portable.copyfile(os.path.join(self.test_root, "amber1"),
fpath)
os.chmod(fpath, 0o555)
os.chown(fpath, -1, 2)
self.pkg("verify", exit=1)
# Make the repository offline.
offline_cb()
# If only attributes on a file are wrong, pkg fix still
# succeeds even if the repository is offline.
self.pkg("fix sysattr")
self.pkg("verify")
online_cb()
self.image_destroy()
# Test other attributes: mode, owner, group and timestamp.
self.image_create(self.durl)
configure_cb()
for p in ("[email protected]","[email protected]", "[email protected]",
"[email protected]", "[email protected]", "[email protected]"):
pfmri = self.plist[p]
self.pkg("install {0}".format(pfmri))
offline_cb()
self.__do_alter_verify(pfmri, parsable=True)
self.pkg("verify --parsable=0 {0}".format(pfmri))
self.pkg("uninstall {0}".format(pfmri))
online_cb()
# If modify the file content locally and its attributes, for the
# editable file delivered with preserve=true, fix doesn't need to
# download the file data.
pfmri = self.plist["[email protected]"]
self.pkg("install {0}".format(pfmri))
self.file_append("amber1", "junk")
offline_cb()
self.__do_alter_verify(pfmri, verbose=True)
self.pkg("uninstall {0}".format(pfmri))
online_cb()
# For editable files delivered with preserve=renamenew or
# preserve=renameold, and non-editable files, fix needs to
# download the file data.
for p in ("[email protected]", "[email protected]", "[email protected]"):
pfmri = self.plist[p]
self.pkg("install {0}".format(pfmri))
self.file_append("amber1", "junk")
offline_cb()
self.__do_alter_verify(pfmri, verbose=True, exit=1)
self.pkg("uninstall {0}".format(pfmri))
online_cb()
# Prepare for next test iteration.
self.image_destroy()
def write(self, path):
"""Write the configuration to the given directory"""
cp = ConfigParser.SafeConfigParser()
# XXX the use of the disabled_auth file can be removed when
# compatibility with the older code is no longer needed
da = ConfigParser.SafeConfigParser()
cp.optionxform = str # preserve option case
# For compatibility, the preferred-publisher is written out
# as the preferred-authority. Modify a copy so that we don't
# change the in-memory copy.
props = self.properties.copy()
try:
del props["preferred-publisher"]
except KeyError:
pass
props["preferred-authority"] = str(self.__publisher_search_order[0])
props["publisher-search-order"] = str(self.__publisher_search_order)
cp.add_section("property")
for p in props:
cp.set("property", p, props[p].encode("utf-8"))
cp.add_section("variant")
for f in self.variants:
cp.set("variant", f, str(self.variants[f]))
cp.add_section("facet")
for f in self.facets:
cp.set("facet", f, str(self.facets[f]))
for prefix in self.__publishers:
pub = self.__publishers[prefix]
section = "authority_%s" % pub.prefix
c = cp
if pub.disabled:
c = da
c.add_section(section)
c.set(section, "alias", str(pub.alias))
c.set(section, "prefix", str(pub.prefix))
c.set(section, "disabled", str(pub.disabled))
c.set(section, "sticky", str(pub.sticky))
repo = pub.selected_repository
# For now, write out "origin" for compatibility with
# older clients in addition to "origins". Older
# clients may drop the "origins" when rewriting the
# configuration, but that doesn't really break
# anything.
c.set(section, "origin", repo.origins[0].uri)
c.set(section, "origins",
str([u.uri for u in repo.origins]))
c.set(section, "mirrors",
str([u.uri for u in repo.mirrors]))
if repo.system_repo:
c.set(section, "sysrepo.uri",
str(repo.system_repo))
c.set(section, "sysrepo.sock_path",
repo.system_repo.socket_path)
else:
c.set(section, "sysrepo.uri", "None")
c.set(section, "sysrepo.sock_path", "None")
#
# For zones, where the reachability of an absolute path
# changes depending on whether you're in the zone or
# not. So we have a different policy: ssl_key and
# ssl_cert are treated as zone root relative.
#
ngz = self.variants.get("variant.opensolaris.zone",
"global") == "nonglobal"
p = str(pub["ssl_key"])
if ngz and self.__imgroot != os.sep and p != "None":
# Trim the imageroot from the path.
if p.startswith(self.__imgroot):
p = p[len(self.__imgroot):]
# XXX this should be per origin or mirror
c.set(section, "ssl_key", p)
p = str(pub["ssl_cert"])
if ngz and self.__imgroot != os.sep and p != "None":
if p.startswith(self.__imgroot):
p = p[len(self.__imgroot):]
# XXX this should be per origin or mirror
c.set(section, "ssl_cert", p)
# XXX this should really be client_uuid, but is being
# left with this name for compatibility with older
# clients.
c.set(section, "uuid", str(pub.client_uuid))
# Write selected repository data.
# XXX this is temporary until a switch to a more
# expressive configuration format is made.
repo = pub.selected_repository
repo_data = {
"collection_type": repo.collection_type,
"description": repo.description,
"legal_uris": [u.uri for u in repo.legal_uris],
"name": repo.name,
"refresh_seconds": repo.refresh_seconds,
"registered": repo.registered,
"registration_uri": repo.registration_uri,
"related_uris": [u.uri for u in repo.related_uris],
"sort_policy": repo.sort_policy,
}
for key, val in repo_data.iteritems():
c.set(section, "repo.%s" % key, str(val))
# XXX Child images
for afile, acp in [(CFG_FILE, cp), (DA_FILE, da)]:
thefile = os.path.join(path, afile)
if len(acp.sections()) == 0:
if os.path.exists(thefile):
portable.remove(thefile)
continue
try:
f = open(thefile, "w")
except EnvironmentError, e:
if e.errno == errno.EACCES:
raise api_errors.PermissionsException(
e.filename)
if e.errno == errno.EROFS:
raise api_errors.ReadOnlyFileSystemException(
e.filename)
raise
acp.write(f)
def tearDown(self):
portable.remove(self.test_fn)
portable.remove(self.test_fn2)
os.rmdir(self.test_path)
os.rmdir(self.unsup_test_path)
gzfobj = tar_stream.extractfile(info)
fpath = os.path.join(tmpdir, info.name)
outfile = open(fpath, "wb")
gunzip_from_stream(gzfobj, outfile)
outfile.close()
gzfobj.close()
else:
# We want to keep the files compressed
# on disk.
tar_stream.extract_to(info, tmpdir, info.name)
# Copy the file into place (rename can cause a cross-
# link device failure) and then remove the original.
src = os.path.join(tmpdir, info.name)
shutil.copy(src, os.path.join(destdir, info.name))
portable.remove(src)
tracker.download_add_progress(1, cshashes[info.name][2])
except KeyboardInterrupt:
raise
except:
abort(err=_("Unable to extract file: %s") % info.name)
shutil.rmtree(tmpdirs.pop(), True)
tar_stream.close()
f.close()
def list_newest_fmris(fmri_list):
"""List the provided fmris."""
pfile.seek(0)
# Write all of the existing entries in the catalog
# into the tempfile. Then append the new lines at the
# end.
try:
for entry in pfile:
if entry == pkgstr:
raise CatalogException(
"Package %s is already in "
"the catalog" % pkgstr)
else:
tfile.write(entry)
tfile.write(pkgstr)
except Exception:
portable.remove(tmpfile)
raise
# Close our open files
pfile.close()
tfile.close()
# Set the permissions on the tempfile correctly.
# Mkstemp creates files as 600. Rename the new
# cataog on top of the old one.
try:
os.chmod(tmpfile, self.file_mode)
portable.rename(tmpfile, self.catalog_file)
except EnvironmentError:
portable.remove(tmpfile)
raise
def test_06_download(self):
"""Test that pkg fix won't try to download all data for
files that fail verification when the data is not going
to be used."""
# If only attributes are wrong and no local modification
# is on the file content, fix doesn't need to download the
# file data.
# Test the system attribute.
# Need to create an image in /var/tmp since sysattrs don't work
# in tmpfs.
old_img_path = self.img_path()
self.set_img_path(tempfile.mkdtemp(prefix="test-suite",
dir="/var/tmp"))
self.image_create(self.durl)
self.pkg("install sysattr")
self.pkg("verify")
fpath = os.path.join(self.img_path(), "amber1")
# Need to get creative here to remove the system attributes
# since you need the sys_linkdir privilege which we don't have:
# see run.py:393
# So we re-create the file with correct owner and mode and the
# only thing missing are the sysattrs.
portable.remove(fpath)
portable.copyfile(os.path.join(self.test_root, "amber1"), fpath)
os.chmod(fpath, 0o555)
os.chown(fpath, -1, 2)
self.pkg("verify", exit=1)
# Make the repository offline.
self.dc.stop()
# If only attributes on a file are wrong, pkg fix still
# succeeds even if the repository is offline.
self.pkg("fix sysattr")
self.pkg("verify")
self.dc.start()
shutil.rmtree(self.img_path())
# Test other attributes: mode, owner, group and timestamp.
self.image_create(self.durl)
for p in ("[email protected]", "[email protected]", "[email protected]",
"[email protected]", "[email protected]", "[email protected]"):
pfmri = self.plist[p]
self.pkg("install {0}".format(pfmri))
self.dc.stop()
self.__do_alter_verify(pfmri, parsable=True)
self.pkg("verify --parsable=0 {0}".format(pfmri))
self.pkg("uninstall {0}".format(pfmri))
self.dc.start()
# If modify the file content locally and its attributes, for the
# editable file delivered with preserve=true, fix doesn't need to
# download the file data.
pfmri = self.plist["[email protected]"]
self.pkg("install {0}".format(pfmri))
self.file_append("amber1", "junk")
self.dc.stop()
self.__do_alter_verify(pfmri, verbose=True)
self.pkg("uninstall {0}".format(pfmri))
self.dc.start()
# For editable files delivered with preserve=renamenew or
# preserve=renameold, and non-editable files, fix needs to
# download the file data.
for p in ("[email protected]", "[email protected]", "[email protected]"):
pfmri = self.plist[p]
self.pkg("install {0}".format(pfmri))
self.file_append("amber1", "junk")
self.dc.stop()
self.__do_alter_verify(pfmri, verbose=True, exit=1)
self.pkg("uninstall {0}".format(pfmri))
self.dc.start()
def _recv_updates(filep, path, cts):
"""A static method that takes a file-like object, a path, and a
timestamp. It reads a stream as an incoming updatelog and
modifies the catalog on disk."""
if not os.path.exists(path):
os.makedirs(path)
# Build a list of FMRIs that this update would add, check to
# make sure that they aren't present in the catalog, then
# append the fmris.
mts = catalog.ts_to_datetime(cts)
cts = mts
pts = mts
added = 0
npkgs = 0
add_lines = []
unknown_lines = []
bad_fmri = None
attrs = {}
for s in filep:
l = s.split(None, 3)
if len(l) < 4:
continue
elif l[2] not in catalog.known_prefixes:
# Add unknown line directly to catalog.
# This can be post-processed later, when it
# becomes known.
#
# XXX Notify user that unknown entry was added?
ts = catalog.ts_to_datetime(l[1])
if ts > cts:
if ts > mts:
pts = mts
mts = ts
line = "{0} {1}\n".format(l[2], l[3])
unknown_lines.append(line)
elif l[0] == "+":
# This is a known entry type.
# Create a list of FMRIs to add, since
# additional inspection is required
ts = catalog.ts_to_datetime(l[1])
if ts > cts:
if ts > mts:
pts = mts
mts = ts
# The format for C and V records
# is described in the Catalog's
# docstring.
if l[2] in tuple("CV"):
try:
f = fmri.PkgFmri(l[3])
except fmri.IllegalFmri as e:
bad_fmri = e
mts = pts
continue
line = "{0} {1} {2} {3}\n".format(
l[2], "pkg", f.pkg_name, f.version)
add_lines.append(line)
added += 1
# If we got a parse error on FMRIs and transfer
# wasn't truncated, raise a retryable transport
if bad_fmri:
raise bad_fmri
# Verify that they aren't already in the catalog
catpath = os.path.normpath(os.path.join(path, "catalog"))
tmp_num, tmpfile = tempfile.mkstemp(dir=path)
tfile = os.fdopen(tmp_num, 'w')
try:
pfile = file(catpath, "rb")
except IOError as e:
if e.errno == errno.ENOENT:
# Creating an empty file
file(catpath, "wb").close()
pfile = file(catpath, "rb")
else:
tfile.close()
portable.remove(tmpfile)
raise
pfile.seek(0)
for c in pfile:
if c[0] in tuple("CV"):
npkgs += 1
if c in add_lines:
pfile.close()
tfile.close()
portable.remove(tmpfile)
raise UpdateLogException(
"Package {0} is already in the catalog".format(c))
tfile.write(c)
# Write the new entries to the catalog
tfile.seek(0, os.SEEK_END)
tfile.writelines(add_lines)
if len(unknown_lines) > 0:
tfile.writelines(unknown_lines)
tfile.close()
pfile.close()
os.chmod(tmpfile, catalog.ServerCatalog.file_mode)
portable.rename(tmpfile, catpath)
# Now re-write npkgs and Last-Modified in attributes file
afile = file(os.path.normpath(os.path.join(path, "attrs")), "r")
attrre = re.compile('^S ([^:]*): (.*)')
for entry in afile:
m = attrre.match(entry)
if m != None:
attrs[m.group(1)] = m.group(2)
afile.close()
# Update the attributes we care about
attrs["npkgs"] = npkgs + added
attrs["Last-Modified"] = mts.isoformat()
# Write attributes back out
apath = os.path.normpath(os.path.join(path, "attrs"))
tmp_num, tmpfile = tempfile.mkstemp(dir=path)
tfile = os.fdopen(tmp_num, 'w')
for a in attrs.keys():
s = "S {0}: {1}\n".format(a, attrs[a])
tfile.write(s)
tfile.close()
os.chmod(tmpfile, catalog.ServerCatalog.file_mode)
portable.rename(tmpfile, apath)
return True