def VerifyUpdate(directory):
"""
Verify the update in the directory is valid -- the manifest
is sane, any signature is valid, the package files necessary to
update are present, and have a valid checksum. Returns either
a file object if it's valid (the file object is locked), None
if it doesn't exist, or it raises an exception -- one of
UpdateIncompleteCacheException or UpdateInvalidCacheException --
if necessary.
"""
import fcntl
# First thing we do is get the systen configuration and
# systen manifest
conf = Configuration.Configuration()
mani = conf.SystemManifest()
# Next, let's see if the directory exists.
if not os.path.exists(directory):
return None
# Open up the manifest file. Assuming it exists.
try:
mani_file = open(directory + "/MANIFEST", "r+")
except:
# Doesn't exist. Or we can't get to it, which would be weird.
return None
# Let's try getting an exclusive lock on the manifest
try:
fcntl.lockf(mani_file, fcntl.LOCK_EX | fcntl.LOCK_NB, 0, 0)
except:
# Well, if we can't acquire the lock, someone else has it.
# Throw an incomplete exception
raise UpdateBusyCacheException("Cache directory %s is being modified" %
directory)
# We always want a valid signature for an update.
cached_mani = Manifest.Manifest(require_signature=True)
try:
cached_mani.LoadFile(mani_file)
except Exception as e:
# If we got an exception, it's invalid.
log.error("Could not load cached manifest file: %s" % str(e))
raise UpdateInvalidCacheException
# First easy thing to do: look for the SEQUENCE file.
try:
cached_sequence = open(directory + "/SEQUENCE", "r").read().rstrip()
except (IOError, Exception) as e:
log.error("Could not sequence file in cache directory %s: %s" %
(directory, str(e)))
raise UpdateIncompleteCacheException(
"Cache directory %s does not have a sequence file" % directory)
# Now let's see if the sequence matches us.
if cached_sequence != mani.Sequence():
log.error("Cached sequence, %s, does not match system sequence, %s" %
(cached_sequence, mani.Sequence()))
raise UpdateInvalidCacheException(
"Cached sequence does not match system sequence")
# Second easy thing to do: if there is a SERVER file, make sure it's the same server
# name we're using
cached_server = "default"
try:
cached_server = open(directory + "/SERVER", "r").read().rstrip()
except (IOError, Exception) as e:
log.debug("Could not open SERVER file in cache direcory %s: %s" %
(directory, str(e)))
cached_server = "default"
if cached_server != conf.UpdateServerName():
log.error(
"Cached server, %s, does not match system update server, %s" %
(cached_server, conf.UpdateServerName()))
raise UpdateInvalidCacheException(
"Cached server name does not match system update server")
# Next thing to do is go through the manifest, and decide which package files we need.
diffs = Manifest.DiffManifests(mani, cached_mani)
# This gives us an array to examine.
# All we care about for verification is the packages
if "Packages" in diffs:
for (pkg, op, old) in diffs["Packages"]:
if op == "delete":
# Deleted package, so we don't need to do any verification here
continue
if op == "install":
# New package, being installed, so we need the full package
cur_vers = None
if op == "upgrade":
# Package being updated, so we can look for the delta package.
cur_vers = old.Version()
new_vers = pkg.Version()
# This is slightly redundant -- if cur_vers is None, it'll check
# the same filename twice.
if not os.path.exists(directory + "/" + pkg.FileName()) and \
not os.path.exists(directory + "/" + pkg.FileName(cur_vers)):
# Neither exists, so incoplete
log.error("Cache %s directory missing files for package %s" %
(directory, pkg.Name()))
raise UpdateIncompleteCacheException(
"Cache directory %s missing files for package %s" %
(directory, pkg.Name()))
# Okay, at least one of them exists.
# Let's try the full file first
try:
with open(directory + "/" + pkg.FileName()) as f:
if pkg.Checksum():
cksum = Configuration.ChecksumFile(f)
if cksum == pkg.Checksum():
continue
else:
continue
except:
pass
if cur_vers is None:
e = "Cache directory %s missing files for package %s" % (
directory, pkg.Name())
log.error(e)
raise UpdateIncompleteCacheException(e)
# Now we try the delta file
# To do that, we need to find the right dictionary in the pkg
upd_cksum = None
update = pkg.Update(cur_vers)
if update and update.Checksum():
upd_cksum = update.Checksum()
try:
with open(directory + "/" + pkg.FileName(cur_vers)) as f:
cksum = Configuration.ChecksumFile(f)
if upd_cksum != cksum:
update = None
except:
update = None
if update is None:
# If we got here, we are missing this file
log_msg = "Cache directory %s is missing package %s" % (
directory, pkg.Name())
log.error(log_msg)
raise UpdateIncompleteCacheException(log_msg)
# And end that loop
# And if we got here, then we have found all of the packages, the manifest is fine,
# and the sequence tag is correct.
mani_file.seek(0)
return mani_file
def PendingUpdatesChanges(directory):
"""
Return a list (a la CheckForUpdates handler right now) of
changes between the currently installed system and the
downloaded contents in <directory>. If <directory>'s values
are incomplete or invalid for whatever reason, return
None. "Incomplete" means a necessary file for upgrading
from the current system is not present; "Invalid" means that
one part of it is invalid -- manifest is not valid, signature isn't
valid, checksum for a file is invalid, or the stashed sequence
number does not match the current system's sequence.
"""
mani_file = None
conf = Configuration.Configuration()
try:
mani_file = VerifyUpdate(directory)
except UpdateBusyCacheException:
log.debug("Cache directory %s is busy, so no update available" %
directory)
raise
except (UpdateIncompleteCacheException, UpdateInvalidCacheException) as e:
log.error(str(e))
RemoveUpdate(directory)
raise
except BaseException as e:
log.error(
"Got exception %s while trying to determine pending updates" %
str(e))
raise
if mani_file:
new_manifest = Manifest.Manifest(require_signature=True)
try:
new_manifest.LoadFile(mani_file)
except ManifestInvalidSignature as e:
log.error("Invalid signature in cached manifest: %s" % str(e))
raise
# This returns a set of differences.
# But we shouldn't rely on it until we can look at what we've
# actually downloaded. To do that, we need to look at any
# package differences (diffs["Packages"]), and check the
# updates if that's what got downloaded.
# By definition, if there are no Packages differences, a reboot
# isn't required.
diffs = Manifest.DiffManifests(conf.SystemManifest(), new_manifest)
if len(diffs) == 0:
return None
if "Packages" in diffs:
reboot = False
# Look through the install/upgrade packages
for pkg, op, old in diffs["Packages"]:
if op == "delete":
continue
if op == "install":
if pkg.RequiresReboot() == True:
reboot = True
if op == "upgrade":
# A bit trickier.
upd = pkg.Update(old.Version())
update_fname = os.path.join(directory,
pkg.FileName(old.Version()))
if upd and os.path.exists(update_fname):
if upd.RequiresReboot() == True:
reboot = True
else:
# Have to assume the full package exists
if pkg.RequiresReboot() == True:
reboot = True
else:
reboot = False
if len(diffs) == 0:
return None
diffs["Reboot"] = reboot
return diffs
def CheckForUpdates(handler=None,
train=None,
cache_dir=None,
diff_handler=None):
"""
Check for an updated manifest. If cache_dir is none, then we try
to download just the latest manifest for the given train, and
compare it to the current system. If cache_dir is set, then we
use the manifest in that directory.
"""
conf = Configuration.Configuration()
new_manifest = None
if cache_dir:
try:
mfile = VerifyUpdate(cache_dir)
if mfile is None:
return None
except UpdateBusyCacheException:
log.debug("Cache directory %s is busy, so no update available" %
cache_dir)
return None
except (UpdateIncompleteCacheException,
UpdateInvalidCacheException) as e:
log.error(
"CheckForUpdate(train = %s, cache_dir = %s): Got exception %s, removing cache"
% (train, cache_dir, str(e)))
RemoveUpdate(cache_dir)
return None
except BaseException as e:
log.error(
"CheckForUpdate(train=%s, cache_dir = %s): Got exception %s" %
(train, cache_dir, str(e)))
raise e
# We always want a valid signature when doing an update
new_manifest = Manifest.Manifest(require_signature=True)
try:
new_manifest.LoadFile(mfile)
except Exception as e:
log.error("Could not load manifest due to %s" % str(e))
raise e
else:
try:
new_manifest = conf.FindLatestManifest(train=train,
require_signature=True)
except Exception as e:
log.error("Could not find latest manifest due to %s" % str(e))
if new_manifest is None:
raise UpdateManifestNotFound("Manifest could not be found!")
# If new_manifest is not the requested train, then we don't have an update to do
if train and train != new_manifest.Train():
log.debug(
"CheckForUpdate(train = %s, cache_dir = %s): Wrong train in caache (%s)"
% (train, cache_dir, new_manifest.Train()))
return None
diffs = Manifest.DiffManifests(conf.SystemManifest(), new_manifest)
if diffs is None or len(diffs) == 0:
return None
log.debug("CheckForUpdate: diffs = %s" % diffs)
reboot = False
if handler and "Packages" in diffs:
for (pkg, op, old) in diffs["Packages"]:
handler(op, pkg, old)
# We attempt to see if the update requires a reboot.
# This is only tentative -- if we can't download a delta package,
# and that's what allows the update to avoid a reboot, a reboot
# is going to be required.
if op == "install":
if pkg.RequiresReboot() == True:
reboot = True
elif op == "upgrade":
upd = pkg.Update(old.Version())
if upd:
if upd.RequiresReboot():
reboot = True
elif pkg.RequiresReboot():
reboot = True
diffs["Reboot"] = reboot
if diff_handler:
diff_handler(diffs)
return new_manifest
def DownloadUpdate(train, directory, get_handler=None, check_handler=None):
"""
Download, if necessary, the LATEST update for train; download
delta packages if possible. Checks to see if the existing content
is the right version. In addition to the current caching code, it
will also stash the current sequence when it downloads; this will
allow it to determine if a reboot into a different boot environment
has happened. This will remove the existing content if it decides
it has to redownload for any reason.
"""
import shutil
import fcntl
conf = Configuration.Configuration()
mani = conf.SystemManifest()
# First thing, let's get the latest manifest
try:
latest_mani = conf.FindLatestManifest(train, require_signature=True)
except ManifestInvalidSignature as e:
log.error("Latest manifest has invalid signature: %s" % str(e))
return False
if latest_mani is None:
# This probably means we have no network. Which means we have
# to trust what we've already downloaded, if anything.
log.error("Unable to find latest manifest for train %s" % train)
try:
VerifyUpdate(directory)
log.debug("Possibly with no network, cached update looks good")
return True
except:
log.debug(
"Possibly with no network, either no cached update or it is bad"
)
return False
cache_mani = Manifest.Manifest(require_signature=True)
try:
mani_file = VerifyUpdate(directory)
if mani_file:
cache_mani.LoadFile(mani_file)
if cache_mani.Sequence() == latest_mani.Sequence():
# Woohoo!
mani_file.close()
log.debug(
"DownloadUpdate: Cache directory has latest manifest")
return True
# Not the latest
mani_file.close()
mani_file = None
except UpdateBusyCacheException:
log.debug("Cache directory %s is busy, so no update available" %
directory)
return False
except (UpdateIncompleteCacheException, UpdateInvalidCacheException,
ManifestInvalidSignature) as e:
# It's incomplete, so we need to remove it
log.error("DownloadUpdate(%s, %s): Got exception %s; removing cache" %
(train, directory, str(e)))
except BaseException as e:
log.error("Got exception %s while trying to prepare update cache" %
str(e))
raise e
# If we're here, then we don't have a (valid) cached update.
log.debug("Removing invalid or incomplete cached update")
RemoveUpdate(directory)
try:
os.makedirs(directory)
except BaseException as e:
log.error("Unable to create directory %s: %s" % (directory, str(e)))
return False
try:
mani_file = open(directory + "/MANIFEST", "wxb")
except (IOError, Exception) as e:
log.error("Unale to create manifest file in directory %s" %
(directory, str(e)))
return False
try:
fcntl.lockf(mani_file, fcntl.LOCK_EX | fcntl.LOCK_NB, 0, 0)
except (IOError, Exception) as e:
log.debug("Unable to lock manifest file: %s" % str(e))
mani_file.close()
return False
# Find out what differences there are
diffs = Manifest.DiffManifests(mani, latest_mani)
if diffs is None or len(diffs) == 0:
log.debug("DownloadUpdate: No update available")
# Remove the cache directory and empty manifest file
RemoveUpdate(directory)
return False
log.debug("DownloadUpdate: diffs = %s" % diffs)
download_packages = []
reboot_required = True
if "Reboot" in diffs:
reboot_required = diffs["Reboot"]
if "Packages" in diffs:
for pkg, op, old in diffs["Packages"]:
if op == "delete":
continue
log.debug("DownloadUpdate: Will %s package %s" % (op, pkg.Name()))
download_packages.append(pkg)
log.debug("Update does%s seem to require a reboot" %
"" if reboot_required else " not")
# Next steps: download the package files.
for indx, pkg in enumerate(download_packages):
# This is where we find out for real if a reboot is required.
# To do that, we may need to know which update was downloaded.
if check_handler:
check_handler(indx + 1, pkg=pkg, pkgList=download_packages)
pkg_file = conf.FindPackageFile(pkg,
save_dir=directory,
handler=get_handler)
if pkg_file is None:
log.error("Could not download package file for %s" % pkg.Name())
RemoveUpdate(directory)
return False
# Almost done: get a changelog if one exists for the train
# If we can't get it, we don't care.
conf.GetChangeLog(train, save_dir=directory, handler=get_handler)
# Then save the manifest file.
latest_mani.StoreFile(mani_file)
# Create the SEQUENCE file.
with open(directory + "/SEQUENCE", "w") as f:
f.write("%s" % conf.SystemManifest().Sequence())
# And create the SERVER file.
with open(directory + "/SERVER", "w") as f:
f.write("%s" % conf.UpdateServerName())
# Then return True!
mani_file.close()
return True
def GetUpdateChanges(old_manifest, new_manifest, cache_dir=None):
"""
This is used by both PendingUpdatesChanges() and CheckForUpdates().
The difference between the two is that the latter doesn't necessarily
have a cache directory, so if cache_dir is none, we have to assume the
update package exists.
This returns a dictionary that will have at least "Reboot" as a key.
"""
def MergeServiceList(base_list, new_list):
"""
Merge new_list into base_list.
For each service in new_list (which is a dictionary),
if the value is True, add it to base_list.
If new_list is an array, simply add each item to
base_list; if it's a dict, we check the value.
"""
if new_list is None:
return base_list
if isinstance(new_list, list):
for svc in new_list:
if not svc in base_list:
base_list.append(svc)
elif isinstance(new_list, dict):
for svc, val in new_list.iteritems():
if val:
if not svc in base_list:
base_list.append(svc)
return base_list
svcs = []
diffs = Manifest.DiffManifests(old_manifest, new_manifest)
if len(diffs) == 0:
return None
reboot = False
if REQUIRE_REBOOT:
reboot = True
if "Packages" in diffs:
# Look through the install/upgrade packages
for pkg, op, old in diffs["Packages"]:
if op == "delete":
continue
if op == "install":
if pkg.RequiresReboot() == True:
reboot = True
else:
pkg_services = pkg.RestartServices()
if pkg_services:
svcs = MergeServiceList(svcs, pkg_services)
elif op == "upgrade":
# A bit trickier.
# If there is a list of services to restart, the update
# path wants to look at that rather than the requires reboot.
# However, one service could be to reboot (this is handled
# below).
upd = pkg.Update(old.Version())
if cache_dir:
update_fname = os.path.join(cache_dir,
pkg.FileName(old.Version()))
else:
update_fname = None
if upd and (update_fname is None
or os.path.exists(update_fname)):
pkg_services = upd.RestartServices()
if pkg_services:
svcs = MergeServiceList(svcs, pkg_services)
else:
if upd.RequiresReboot() == True:
reboot = True
else:
# Have to assume the full package exists
if pkg.RequiresReboot() == True:
reboot = True
else:
pkg_services = pkg.RestartServices()
if pkg_services:
svcs = MergeServiceList(svcs, pkg_services)
else:
reboot = False
if len(diffs) == 0:
return None
if not reboot and svcs:
if not VerifyServices(svcs):
reboot = True
else:
diffs["Restart"] = svcs
diffs["Reboot"] = reboot
return diffs