def CheckForUpdates(root = None, handler = None):
"""
Check for an updated manifest.
Very simple, uses the configuration module.
Returns the new manifest if there is an update,
and None otherwise.
(It determines if there is an update if the latest-found
manifeset's sequence number is larger than the current
sequence number.)
The optional argument handler is a function that
will be called for each difference in the new manifest
(if there is one); it will be called with three
arguments: operation, package, old package.
operation will be "delete", "upgrade", or "install";
old package will be None for delete and install.
"""
conf = Configuration.Configuration(root)
cur = conf.SystemManifest()
m = conf.FindLatestManifest()
print >> sys.stderr, "Current sequence = %d, available sequence = %d" % (cur.Sequence(), m.Sequence() if m is not None else 0)
if m is not None and m.Sequence() > cur.Sequence():
if handler is not None:
diffs = Manifest.CompareManifests(cur, m)
for (pkg, op, old) in diffs:
handler(op, pkg, old)
return m
return None
def PendingUpdates(directory):
"""
Return a list (a la CheckForUpdates handler right now) of
changes between the currently installed system and the
downloaded contents in <directory>. If <directory>'s values
are incomplete or invalid for whatever reason, return
None. "Incomplete" means a necessary file for upgrading
from the current system is not present; "Invalid" means that
one part of it is invalid -- manifest is not valid, signature isn't
valid, checksum for a file is invalid, or the stashed sequence
number does not match the current system's sequence.
"""
mani_file = None
conf = Configuration.Configuration()
try:
mani_file = VerifyUpdate(directory)
except UpdateBusyCacheException:
log.debug("Cache directory %s is busy, so no update available" %
directory)
return None
except (UpdateIncompleteCacheException, UpdateInvalidCacheException) as e:
log.error(str(e))
RemoveUpdate(directory)
return None
except BaseException as e:
log.error(
"Got exception %s while trying to determine pending updates" %
str(e))
return None
if mani_file:
new_manifest = Manifest.Manifest()
new_manifest.LoadFile(mani_file)
diffs = Manifest.CompareManifests(conf.SystemManifest(), new_manifest)
return diffs
return None
def CheckForUpdates(root = None, handler = None):
"""
Check for an updated manifest.
Very simple, uses the configuration module.
Returns the new manifest if there is an update,
and None otherwise.
(It determines if there is an update if the latest-found
manifest contains differences from the current system
manifest.)
The optional argument handler is a function that
will be called for each difference in the new manifest
(if there is one); it will be called with three
arguments: operation, package, old package.
operation will be "delete", "upgrade", or "install";
old package will be None for delete and install.
"""
conf = Configuration.Configuration(root)
cur = conf.SystemManifest()
m = conf.FindLatestManifest()
log.debug("Current sequence = %s, available sequence = %s" % (cur.Sequence(), m.Sequence()
if m is not None else "None"))
if m is None:
raise ValueError("Manifest could not be found!")
diffs = Manifest.CompareManifests(cur, m)
update = False
for (pkg, op,old) in diffs:
update = True
if handler is not None:
handler(op, pkg, old)
return m if update else None
def CheckForUpdates(root=None, handler=None, train=None, cache_dir=None):
"""
Check for an updated manifest.
Very simple, uses the configuration module.
Returns the new manifest if there is an update,
and None otherwise.
(It determines if there is an update if the latest-found
manifest contains differences from the current system
manifest.)
The optional argument handler is a function that
will be called for each difference in the new manifest
(if there is one); it will be called with three
arguments: operation, package, old package.
operation will be "delete", "upgrade", or "install";
old package will be None for delete and install.
The optional cache_dir parameter indicates that it
should look in that directory first, rather than
going over the network. (Unlike the similar code
in freenas-update, this will not [at this time]
download the package files and store them in
cache_dir.)
"""
conf = Configuration.Configuration(root)
cur = conf.SystemManifest()
m = None
# Let's check cache_dir if it is set
if cache_dir and (not train or train == cur.Train()):
if os.path.exists(cache_dir + "/MANIFEST"):
# Okay, let's load it up.
m = Manifest.Manifest()
try:
m.LoadPath(cache_dir + "/MANIFEST")
if m.Train() != cur.Train():
# Should we get rid of the cache?
m = None
except:
m = None
if m is None:
m = conf.FindLatestManifest(train=train)
log.debug("Current sequence = %s, available sequence = %s" %
(cur.Sequence(), m.Sequence() if m is not None else "None"))
if m is None:
raise ValueError("Manifest could not be found!")
diffs = Manifest.CompareManifests(cur, m)
update = False
for (pkg, op, old) in diffs:
update = True
if handler is not None:
handler(op, pkg, old)
return m if update else None
def VerifyUpdate(directory):
"""
Verify the update in the directory is valid -- the manifest
is sane, any signature is valid, the package files necessary to
update are present, and have a valid checksum. Returns either
a file object if it's valid (the file object is locked), None
if it doesn't exist, or it raises an exception -- one of
UpdateIncompleteCacheException or UpdateInvalidCacheException --
if necessary.
"""
import fcntl
# First thing we do is get the systen configuration and
# systen manifest
conf = Configuration.Configuration()
mani = conf.SystemManifest()
# Next, let's see if the directory exists.
if not os.path.exists(directory):
return None
# Open up the manifest file. Assuming it exists.
try:
mani_file = open(directory + "/MANIFEST", "r+")
except:
# Doesn't exist. Or we can't get to it, which would be weird.
return None
# Let's try getting an exclusive lock on the manifest
try:
fcntl.lockf(mani_file, fcntl.LOCK_EX | fcntl.LOCK_NB, 0, 0)
except:
# Well, if we can't acquire the lock, someone else has it.
# Throw an incomplete exception
raise UpdateBusyCacheException("Cache directory %s is being modified" %
directory)
cached_mani = Manifest.Manifest()
try:
cached_mani.LoadFile(mani_file)
except Exception as e:
# If we got an exception, it's invalid.
log.error("Could not load cached manifest file: %s" % str(e))
raise UpdateInvalidCacheException
# First easy thing to do: look for the SEQUENCE file.
try:
cached_sequence = open(directory + "/SEQUENCE", "r").read().rstrip()
except (IOError, Exception) as e:
log.error("Could not sequence file in cache directory %s: %s" %
(directory, str(e)))
raise UpdateIncompleteCacheException(
"Cache directory %s does not have a sequence file" % directory)
# Now let's see if the sequence matches us.
if cached_sequence != mani.Sequence():
log.error("Cached sequence, %s, does not match system sequence, %s" %
(cached_sequence, mani.Sequence()))
raise UpdateInvalidCacheException(
"Cached sequence does not match system sequence")
# Next thing to do is go through the manifest, and decide which package files we need.
diffs = Manifest.CompareManifests(mani, cached_mani)
# This gives us an array to examine.
for (pkg, op, old) in diffs:
if op == "delete":
# Deleted package, so we don't need to do any verification here
continue
if op == "install":
# New package, being installed, so we need the full package
cur_vers = None
if op == "upgrade":
# Package being updated, so we can look for the delta package.
cur_vers = old.Version()
new_vers = pkg.Version()
# This is slightly redundant -- if cur_vers is None, it'll check
# the same filename twice.
if not os.path.exists(directory + "/" + pkg.FileName()) and \
not os.path.exists(directory + "/" + pkg.FileName(cur_vers)):
# Neither exists, so incoplete
log.error("Cache %s directory missing files for package %s" %
(directory, pkg.Name()))
raise UpdateIncompleteCacheException(
"Cache directory %s missing files for package %s" %
(directory, pkg.Name()))
# Okay, at least one of them exists.
# Let's try the full file first
try:
with open(directory + "/" + pkg.FileName()) as f:
if pkg.Checksum():
cksum = Configuration.ChecksumFile(f)
if cksum == pkg.Checksum():
continue
else:
continue
except:
pass
if cur_vers is None:
e = "Cache directory %s missing files for package %s" % (
directory, pkg.Name())
log.error(e)
raise UpdateIncompleteCacheException(e)
# Now we try the delta file
# To do that, we need to find the right dictionary in the pkg
upd_cksum = None
found = False
for update_dict in pkg.Updates():
if update_dict[Package.VERSION_KEY] == cur_vers:
if Package.CHECKSUM_KEY in update_dict:
upd_cksum = update_dict[Package.CHECKSUM_KEY]
try:
with open(directory + "/" +
pkg.FileName(cur_vers)) as f:
cksum = Configuration.ChecksumFile(f)
if upd_cksum == cksum:
found = True
break
except:
pass
else:
found = True
break
if found is False:
# If we got here, we are missing this file
log_msg = "Cache directory %s is missing package %s" % (directory,
pkg.Name())
log.error(log_msg)
raise UpdateIncompleteCacheException(log_msg)
# And if we got here, then we have found all of the packages, the manifest is fine,
# and the sequence tag is correct.
mani_file.seek(0)
return mani_file
def DownloadUpdate(train, directory, get_handler=None, check_handler=None):
"""
Download, if necessary, the LATEST update for train; download
delta packages if possible. Checks to see if the existing content
is the right version. In addition to the current caching code, it
will also stash the current sequence when it downloads; this will
allow it to determine if a reboot into a different boot environment
has happened. This will remove the existing content if it decides
it has to redownload for any reason.
"""
import shutil
import fcntl
conf = Configuration.Configuration()
mani = conf.SystemManifest()
# First thing, let's get the latest manifest
latest_mani = conf.FindLatestManifest(train)
if latest_mani is None:
# That really shouldnt happen
log.error("Unable to find latest manifest for train %s" % train)
return False
cache_mani = Manifest.Manifest()
try:
mani_file = VerifyUpdate(directory)
if mani_file:
cache_mani.LoadFile(mani_file)
if cache_mani.Sequence() == latest_mani.Sequence():
# Woohoo!
mani_file.close()
return True
# Not the latest
mani_file.close()
RemoveUpdate(directory)
mani_file = None
except UpdateBusyCacheException:
log.debug("Cache directory %s is busy, so no update available" %
directory)
return False
except (UpdateIncompleteCacheException, UpdateInvalidCacheException) as e:
# It's incomplete, so we need to remove it
log.error("DownloadUpdate(%s, %s): Got exception %s; removing cache" %
(train, directory, str(e)))
RemoveUpdate(directory)
except BaseException as e:
log.error("Got exception %s while trying to prepare update cache" %
str(e))
raise e
# If we're here, then we don't have a (valid) cached update.
RemoveUpdate(directory)
try:
os.makedirs(directory)
except BaseException as e:
log.error("Unable to create directory %s: %s" % (directory, str(e)))
return False
try:
mani_file = open(directory + "/MANIFEST", "wxb")
except (IOError, Exception) as e:
log.error("Unale to create manifest file in directory %s" %
(directory, str(e)))
return False
try:
fcntl.lockf(mani_file, fcntl.LOCK_EX | fcntl.LOCK_NB, 0, 0)
except (IOError, Exception) as e:
log.debug("Unable to lock manifest file: %s" % str(e))
mani_file.close()
return False
# Find out what differences there are
diffs = Manifest.CompareManifests(mani, latest_mani)
download_packages = []
for pkg, op, old in diffs:
if op == "delete":
continue
download_packages.append(pkg)
# Next steps: download the package files.
for indx, pkg in enumerate(download_packages):
if check_handler:
check_handler(indx + 1, pkg=pkg, pkgList=download_packages)
pkg_file = conf.FindPackageFile(pkg,
save_dir=directory,
handler=get_handler)
if pkg_file is None:
log.error("Could not download package file for %s" % pkg.Name())
RemoveUpdate(directory)
return False
# Then save the manifest file.
latest_mani.StoreFile(mani_file)
# Create the SEQUENCE file.
with open(directory + "/SEQUENCE", "w") as f:
f.write("%s" % conf.SystemManifest().Sequence())
# Then return True!
mani_file.close()
return True
def Update(root=None,
conf=None,
train=None,
check_handler=None,
get_handler=None,
install_handler=None,
cache_dir=None):
"""
Perform an update. If cache_dir is set, and contains a valid
set of files (MANIFEST exists, and is for the current train, and
the sequence is not the same as the system sequence), it will
use that, rather than downloading. It calls CheckForUpdates() otherwise,
to see if there are any. If there are, then magic happens.
"""
log.debug("Update(root = %s, conf = %s, train = %s, cache_dir = %s)" %
(root, conf, train, cache_dir))
if conf is None:
conf = Configuration.Configuration(root)
deleted_packages = []
process_packages = []
def UpdateHandler(op, pkg, old):
if op == "delete":
deleted_packages.append(pkg)
else:
process_packages.append(pkg)
if check_handler is not None:
check_handler(op, pkg, old)
new_man = None
if cache_dir:
if os.path.exists(cache_dir + "/MANIFEST"):
try:
cur = conf.SystemManifest()
new_man = Manifest.Manifest()
new_man.LoadPath(cache_dir + "/MANIFEST")
if new_man.Train() != cur.Train():
new_man = None
cache_dir = None
else:
diffs = Manifest.CompareManifests(cur, new_man)
for (pkg, op, old) in diffs:
UpdateHandler(op, pkg, old)
conf.SetPackageDir(cache_dir)
except Exception as e:
log.debug("Caught exception %s" % str(e))
new_man = None
cache_dir = None
if new_man is None:
new_man = CheckForUpdates(root, handler=UpdateHandler, train=train)
if new_man is None:
return
if len(deleted_packages) == 0 and len(process_packages) == 0:
# We have a case where a manifest was updated, but we
# don't actually have any changes to the packages. We
# should install the new manifest, and be done -- it
# may have new release notes, or other issues.
# Right now, I'm not quite sure how to do this.
log.debug("Updated manifest but no package differences")
return
# Now we have a list of deleted packages, and a list
# of update/install packages.
# The task is to delete the first set of packages,
# and then run through the others to install/update.
# First, however, we need to get the files.
# We want to use the system configuration, unless one has been
# specified -- we don't want to use the target root's.
if conf is None:
conf = Configuration.Configuration()
# If root is None, then we will try to create a clone
# environment. (If the caller wants to install into the
# current boot environment, set root = "" or "/".)
clone_name = None
mount_point = None
try:
if root is None:
# We clone the existing boot environment to
# "Avatar()-<sequence>"
clone_name = "%s-%s" % (Avatar(), new_man.Sequence())
if CreateClone(clone_name) is False:
log.error("Unable to create boot-environment %s" % clone_name)
raise Exception("Unable to create new boot-environment %s" %
clone_name)
mount_point = MountClone(clone_name)
if mount_point is None:
log.error("Unable to mount boot-environment %s" % clone_name)
DeleteClone(clone_name)
raise Exception("Unable to mount boot-environment %s" %
clone_name)
else:
root = mount_point
else:
mount_point = None
for pkg in deleted_packages:
log.debug("Want to delete package %s" % pkg.Name())
if conf.PackageDB(root).RemovePackageContents(pkg) == False:
log.error("Unable to remove contents package %s" % pkg.Name())
if mount_point:
UnmountClone(clone_name, mount_point)
mount_point = None
DeleteClone(clone_name)
raise Exception("Unable to remove contents for package %s" %
pkg.Name())
conf.PackageDB(root).RemovePackage(pkg.Name())
installer = Installer.Installer(manifest=new_man,
root=root,
config=conf)
installer.GetPackages(process_packages, handler=get_handler)
log.debug("Packages = %s" % installer._packages)
# Now let's actually install them.
# Only change on success
rv = False
if installer.InstallPackages(handler=install_handler) is False:
log.error("Unable to install packages")
else:
new_man.Save(root)
if mount_point is not None:
if UnmountClone(clone_name, mount_point) is False:
log.error("Unable to mount clone enivironment %s" %
clone_name)
else:
mount_point = None
if ActivateClone(clone_name) is False:
log.error("Could not activate clone environment %s" %
clone_name)
else:
# Downloaded package files are open-unlinked, so don't
# have to be cleaned up. If there's a cache directory,
# however, we need to get rid of it.
if cache_dir:
import shutil
if os.path.exists(cache_dir):
try:
shutil.rmtree(cache_dir)
except Exception as e:
# If that doesn't work, for now at least we'll
# simply ignore the error.
log.debug(
"Tried to remove cache directory %s, got exception %s"
% (cache_dir, str(e)))
rv = True
# Start a scrub. We ignore the return value.
RunCommand("/sbin/zpool", ["scrub", "freenas-boot"])
except BaseException as e:
log.error("Update got exception during update: %s" % str(e))
if clone_name:
if mount_point:
# Ignore the error here
UnmountClone(clone_name, mount_point)
DeleteClone(clone_name)
raise e
# Clean up
# That just leaves the clone, which
# we should unmount, and destroy if necessary.
# Unmounting attempts to delete the mount point that was created.
if rv is False:
if clone_name and DeleteClone(clone_name) is False:
log.error("Unable to delete boot environment %s in failure case" %
clone_name)
return rv
def CheckForUpdates(handler=None, train=None, cache_dir=None):
"""
Check for an updated manifest. If cache_dir is none, then we try
to download just the latest manifest for the given train, and
compare it to the current system. If cache_dir is set, then we
use the manifest in that directory.
"""
conf = Configuration.Configuration()
new_manifest = None
if cache_dir:
try:
mfile = VerifyUpdate(cache_dir)
if mfile is None:
return None
except UpdateBusyCacheException:
log.debug("Cache directory %s is busy, so no update available" %
cache_dir)
return None
except (UpdateIncompleteCacheException,
UpdateInvalidCacheException) as e:
log.error(
"CheckForUpdate(train = %s, cache_dir = %s): Got exception %s, removing cache"
% (train, cache_dir, str(e)))
RemoveUpdate(cache_dir)
return None
except BaseException as e:
log.error(
"CheckForUpdate(train=%s, cache_dir = %s): Got exception %s" %
(train, cache_dir, str(e)))
raise e
# We always want a valid signature when doing an update
new_manifest = Manifest.Manifest(require_signature=True)
try:
new_manifest.LoadFile(mfile)
except Exception as e:
log.error("Could not load manifest due to %s" % str(e))
raise e
else:
try:
new_manifest = conf.FindLatestManifest(train=train,
require_signature=True)
except Exception as e:
log.error("Could not find latest manifest due to %s" % str(e))
if new_manifest is None:
raise UpdateManifestNotFound("Manifest could not be found!")
# If new_manifest is not the requested train, then we don't have an update to do
if train and train != new_manifest.Train():
log.debug(
"CheckForUpdate(train = %s, cache_dir = %s): Wrong train in caache (%s)"
% (train, cache_dir, new_manifest.Train()))
return None
diffs = Manifest.CompareManifests(conf.SystemManifest(), new_manifest)
if diffs is None or len(diffs) == 0:
return None
if handler:
for (pkg, op, old) in diffs:
handler(op, pkg, old)
return new_manifest