def CheckForUpdates(root = None, handler = None): """ Check for an updated manifest. Very simple, uses the configuration module. Returns the new manifest if there is an update, and None otherwise. (It determines if there is an update if the latest-found manifeset's sequence number is larger than the current sequence number.) The optional argument handler is a function that will be called for each difference in the new manifest (if there is one); it will be called with three arguments: operation, package, old package. operation will be "delete", "upgrade", or "install"; old package will be None for delete and install. """ conf = Configuration.Configuration(root) cur = conf.SystemManifest() m = conf.FindLatestManifest() print >> sys.stderr, "Current sequence = %d, available sequence = %d" % (cur.Sequence(), m.Sequence() if m is not None else 0) if m is not None and m.Sequence() > cur.Sequence(): if handler is not None: diffs = Manifest.CompareManifests(cur, m) for (pkg, op, old) in diffs: handler(op, pkg, old) return m return None
def PendingUpdates(directory): """ Return a list (a la CheckForUpdates handler right now) of changes between the currently installed system and the downloaded contents in <directory>. If <directory>'s values are incomplete or invalid for whatever reason, return None. "Incomplete" means a necessary file for upgrading from the current system is not present; "Invalid" means that one part of it is invalid -- manifest is not valid, signature isn't valid, checksum for a file is invalid, or the stashed sequence number does not match the current system's sequence. """ mani_file = None conf = Configuration.Configuration() try: mani_file = VerifyUpdate(directory) except UpdateBusyCacheException: log.debug("Cache directory %s is busy, so no update available" % directory) return None except (UpdateIncompleteCacheException, UpdateInvalidCacheException) as e: log.error(str(e)) RemoveUpdate(directory) return None except BaseException as e: log.error( "Got exception %s while trying to determine pending updates" % str(e)) return None if mani_file: new_manifest = Manifest.Manifest() new_manifest.LoadFile(mani_file) diffs = Manifest.CompareManifests(conf.SystemManifest(), new_manifest) return diffs return None
def CheckForUpdates(root = None, handler = None): """ Check for an updated manifest. Very simple, uses the configuration module. Returns the new manifest if there is an update, and None otherwise. (It determines if there is an update if the latest-found manifest contains differences from the current system manifest.) The optional argument handler is a function that will be called for each difference in the new manifest (if there is one); it will be called with three arguments: operation, package, old package. operation will be "delete", "upgrade", or "install"; old package will be None for delete and install. """ conf = Configuration.Configuration(root) cur = conf.SystemManifest() m = conf.FindLatestManifest() log.debug("Current sequence = %s, available sequence = %s" % (cur.Sequence(), m.Sequence() if m is not None else "None")) if m is None: raise ValueError("Manifest could not be found!") diffs = Manifest.CompareManifests(cur, m) update = False for (pkg, op,old) in diffs: update = True if handler is not None: handler(op, pkg, old) return m if update else None
def CheckForUpdates(root=None, handler=None, train=None, cache_dir=None): """ Check for an updated manifest. Very simple, uses the configuration module. Returns the new manifest if there is an update, and None otherwise. (It determines if there is an update if the latest-found manifest contains differences from the current system manifest.) The optional argument handler is a function that will be called for each difference in the new manifest (if there is one); it will be called with three arguments: operation, package, old package. operation will be "delete", "upgrade", or "install"; old package will be None for delete and install. The optional cache_dir parameter indicates that it should look in that directory first, rather than going over the network. (Unlike the similar code in freenas-update, this will not [at this time] download the package files and store them in cache_dir.) """ conf = Configuration.Configuration(root) cur = conf.SystemManifest() m = None # Let's check cache_dir if it is set if cache_dir and (not train or train == cur.Train()): if os.path.exists(cache_dir + "/MANIFEST"): # Okay, let's load it up. m = Manifest.Manifest() try: m.LoadPath(cache_dir + "/MANIFEST") if m.Train() != cur.Train(): # Should we get rid of the cache? m = None except: m = None if m is None: m = conf.FindLatestManifest(train=train) log.debug("Current sequence = %s, available sequence = %s" % (cur.Sequence(), m.Sequence() if m is not None else "None")) if m is None: raise ValueError("Manifest could not be found!") diffs = Manifest.CompareManifests(cur, m) update = False for (pkg, op, old) in diffs: update = True if handler is not None: handler(op, pkg, old) return m if update else None
def VerifyUpdate(directory): """ Verify the update in the directory is valid -- the manifest is sane, any signature is valid, the package files necessary to update are present, and have a valid checksum. Returns either a file object if it's valid (the file object is locked), None if it doesn't exist, or it raises an exception -- one of UpdateIncompleteCacheException or UpdateInvalidCacheException -- if necessary. """ import fcntl # First thing we do is get the systen configuration and # systen manifest conf = Configuration.Configuration() mani = conf.SystemManifest() # Next, let's see if the directory exists. if not os.path.exists(directory): return None # Open up the manifest file. Assuming it exists. try: mani_file = open(directory + "/MANIFEST", "r+") except: # Doesn't exist. Or we can't get to it, which would be weird. return None # Let's try getting an exclusive lock on the manifest try: fcntl.lockf(mani_file, fcntl.LOCK_EX | fcntl.LOCK_NB, 0, 0) except: # Well, if we can't acquire the lock, someone else has it. # Throw an incomplete exception raise UpdateBusyCacheException("Cache directory %s is being modified" % directory) cached_mani = Manifest.Manifest() try: cached_mani.LoadFile(mani_file) except Exception as e: # If we got an exception, it's invalid. log.error("Could not load cached manifest file: %s" % str(e)) raise UpdateInvalidCacheException # First easy thing to do: look for the SEQUENCE file. try: cached_sequence = open(directory + "/SEQUENCE", "r").read().rstrip() except (IOError, Exception) as e: log.error("Could not sequence file in cache directory %s: %s" % (directory, str(e))) raise UpdateIncompleteCacheException( "Cache directory %s does not have a sequence file" % directory) # Now let's see if the sequence matches us. if cached_sequence != mani.Sequence(): log.error("Cached sequence, %s, does not match system sequence, %s" % (cached_sequence, mani.Sequence())) raise UpdateInvalidCacheException( "Cached sequence does not match system sequence") # Next thing to do is go through the manifest, and decide which package files we need. diffs = Manifest.CompareManifests(mani, cached_mani) # This gives us an array to examine. for (pkg, op, old) in diffs: if op == "delete": # Deleted package, so we don't need to do any verification here continue if op == "install": # New package, being installed, so we need the full package cur_vers = None if op == "upgrade": # Package being updated, so we can look for the delta package. cur_vers = old.Version() new_vers = pkg.Version() # This is slightly redundant -- if cur_vers is None, it'll check # the same filename twice. if not os.path.exists(directory + "/" + pkg.FileName()) and \ not os.path.exists(directory + "/" + pkg.FileName(cur_vers)): # Neither exists, so incoplete log.error("Cache %s directory missing files for package %s" % (directory, pkg.Name())) raise UpdateIncompleteCacheException( "Cache directory %s missing files for package %s" % (directory, pkg.Name())) # Okay, at least one of them exists. # Let's try the full file first try: with open(directory + "/" + pkg.FileName()) as f: if pkg.Checksum(): cksum = Configuration.ChecksumFile(f) if cksum == pkg.Checksum(): continue else: continue except: pass if cur_vers is None: e = "Cache directory %s missing files for package %s" % ( directory, pkg.Name()) log.error(e) raise UpdateIncompleteCacheException(e) # Now we try the delta file # To do that, we need to find the right dictionary in the pkg upd_cksum = None found = False for update_dict in pkg.Updates(): if update_dict[Package.VERSION_KEY] == cur_vers: if Package.CHECKSUM_KEY in update_dict: upd_cksum = update_dict[Package.CHECKSUM_KEY] try: with open(directory + "/" + pkg.FileName(cur_vers)) as f: cksum = Configuration.ChecksumFile(f) if upd_cksum == cksum: found = True break except: pass else: found = True break if found is False: # If we got here, we are missing this file log_msg = "Cache directory %s is missing package %s" % (directory, pkg.Name()) log.error(log_msg) raise UpdateIncompleteCacheException(log_msg) # And if we got here, then we have found all of the packages, the manifest is fine, # and the sequence tag is correct. mani_file.seek(0) return mani_file
def DownloadUpdate(train, directory, get_handler=None, check_handler=None): """ Download, if necessary, the LATEST update for train; download delta packages if possible. Checks to see if the existing content is the right version. In addition to the current caching code, it will also stash the current sequence when it downloads; this will allow it to determine if a reboot into a different boot environment has happened. This will remove the existing content if it decides it has to redownload for any reason. """ import shutil import fcntl conf = Configuration.Configuration() mani = conf.SystemManifest() # First thing, let's get the latest manifest latest_mani = conf.FindLatestManifest(train) if latest_mani is None: # That really shouldnt happen log.error("Unable to find latest manifest for train %s" % train) return False cache_mani = Manifest.Manifest() try: mani_file = VerifyUpdate(directory) if mani_file: cache_mani.LoadFile(mani_file) if cache_mani.Sequence() == latest_mani.Sequence(): # Woohoo! mani_file.close() return True # Not the latest mani_file.close() RemoveUpdate(directory) mani_file = None except UpdateBusyCacheException: log.debug("Cache directory %s is busy, so no update available" % directory) return False except (UpdateIncompleteCacheException, UpdateInvalidCacheException) as e: # It's incomplete, so we need to remove it log.error("DownloadUpdate(%s, %s): Got exception %s; removing cache" % (train, directory, str(e))) RemoveUpdate(directory) except BaseException as e: log.error("Got exception %s while trying to prepare update cache" % str(e)) raise e # If we're here, then we don't have a (valid) cached update. RemoveUpdate(directory) try: os.makedirs(directory) except BaseException as e: log.error("Unable to create directory %s: %s" % (directory, str(e))) return False try: mani_file = open(directory + "/MANIFEST", "wxb") except (IOError, Exception) as e: log.error("Unale to create manifest file in directory %s" % (directory, str(e))) return False try: fcntl.lockf(mani_file, fcntl.LOCK_EX | fcntl.LOCK_NB, 0, 0) except (IOError, Exception) as e: log.debug("Unable to lock manifest file: %s" % str(e)) mani_file.close() return False # Find out what differences there are diffs = Manifest.CompareManifests(mani, latest_mani) download_packages = [] for pkg, op, old in diffs: if op == "delete": continue download_packages.append(pkg) # Next steps: download the package files. for indx, pkg in enumerate(download_packages): if check_handler: check_handler(indx + 1, pkg=pkg, pkgList=download_packages) pkg_file = conf.FindPackageFile(pkg, save_dir=directory, handler=get_handler) if pkg_file is None: log.error("Could not download package file for %s" % pkg.Name()) RemoveUpdate(directory) return False # Then save the manifest file. latest_mani.StoreFile(mani_file) # Create the SEQUENCE file. with open(directory + "/SEQUENCE", "w") as f: f.write("%s" % conf.SystemManifest().Sequence()) # Then return True! mani_file.close() return True
def Update(root=None, conf=None, train=None, check_handler=None, get_handler=None, install_handler=None, cache_dir=None): """ Perform an update. If cache_dir is set, and contains a valid set of files (MANIFEST exists, and is for the current train, and the sequence is not the same as the system sequence), it will use that, rather than downloading. It calls CheckForUpdates() otherwise, to see if there are any. If there are, then magic happens. """ log.debug("Update(root = %s, conf = %s, train = %s, cache_dir = %s)" % (root, conf, train, cache_dir)) if conf is None: conf = Configuration.Configuration(root) deleted_packages = [] process_packages = [] def UpdateHandler(op, pkg, old): if op == "delete": deleted_packages.append(pkg) else: process_packages.append(pkg) if check_handler is not None: check_handler(op, pkg, old) new_man = None if cache_dir: if os.path.exists(cache_dir + "/MANIFEST"): try: cur = conf.SystemManifest() new_man = Manifest.Manifest() new_man.LoadPath(cache_dir + "/MANIFEST") if new_man.Train() != cur.Train(): new_man = None cache_dir = None else: diffs = Manifest.CompareManifests(cur, new_man) for (pkg, op, old) in diffs: UpdateHandler(op, pkg, old) conf.SetPackageDir(cache_dir) except Exception as e: log.debug("Caught exception %s" % str(e)) new_man = None cache_dir = None if new_man is None: new_man = CheckForUpdates(root, handler=UpdateHandler, train=train) if new_man is None: return if len(deleted_packages) == 0 and len(process_packages) == 0: # We have a case where a manifest was updated, but we # don't actually have any changes to the packages. We # should install the new manifest, and be done -- it # may have new release notes, or other issues. # Right now, I'm not quite sure how to do this. log.debug("Updated manifest but no package differences") return # Now we have a list of deleted packages, and a list # of update/install packages. # The task is to delete the first set of packages, # and then run through the others to install/update. # First, however, we need to get the files. # We want to use the system configuration, unless one has been # specified -- we don't want to use the target root's. if conf is None: conf = Configuration.Configuration() # If root is None, then we will try to create a clone # environment. (If the caller wants to install into the # current boot environment, set root = "" or "/".) clone_name = None mount_point = None try: if root is None: # We clone the existing boot environment to # "Avatar()-<sequence>" clone_name = "%s-%s" % (Avatar(), new_man.Sequence()) if CreateClone(clone_name) is False: log.error("Unable to create boot-environment %s" % clone_name) raise Exception("Unable to create new boot-environment %s" % clone_name) mount_point = MountClone(clone_name) if mount_point is None: log.error("Unable to mount boot-environment %s" % clone_name) DeleteClone(clone_name) raise Exception("Unable to mount boot-environment %s" % clone_name) else: root = mount_point else: mount_point = None for pkg in deleted_packages: log.debug("Want to delete package %s" % pkg.Name()) if conf.PackageDB(root).RemovePackageContents(pkg) == False: log.error("Unable to remove contents package %s" % pkg.Name()) if mount_point: UnmountClone(clone_name, mount_point) mount_point = None DeleteClone(clone_name) raise Exception("Unable to remove contents for package %s" % pkg.Name()) conf.PackageDB(root).RemovePackage(pkg.Name()) installer = Installer.Installer(manifest=new_man, root=root, config=conf) installer.GetPackages(process_packages, handler=get_handler) log.debug("Packages = %s" % installer._packages) # Now let's actually install them. # Only change on success rv = False if installer.InstallPackages(handler=install_handler) is False: log.error("Unable to install packages") else: new_man.Save(root) if mount_point is not None: if UnmountClone(clone_name, mount_point) is False: log.error("Unable to mount clone enivironment %s" % clone_name) else: mount_point = None if ActivateClone(clone_name) is False: log.error("Could not activate clone environment %s" % clone_name) else: # Downloaded package files are open-unlinked, so don't # have to be cleaned up. If there's a cache directory, # however, we need to get rid of it. if cache_dir: import shutil if os.path.exists(cache_dir): try: shutil.rmtree(cache_dir) except Exception as e: # If that doesn't work, for now at least we'll # simply ignore the error. log.debug( "Tried to remove cache directory %s, got exception %s" % (cache_dir, str(e))) rv = True # Start a scrub. We ignore the return value. RunCommand("/sbin/zpool", ["scrub", "freenas-boot"]) except BaseException as e: log.error("Update got exception during update: %s" % str(e)) if clone_name: if mount_point: # Ignore the error here UnmountClone(clone_name, mount_point) DeleteClone(clone_name) raise e # Clean up # That just leaves the clone, which # we should unmount, and destroy if necessary. # Unmounting attempts to delete the mount point that was created. if rv is False: if clone_name and DeleteClone(clone_name) is False: log.error("Unable to delete boot environment %s in failure case" % clone_name) return rv
def CheckForUpdates(handler=None, train=None, cache_dir=None): """ Check for an updated manifest. If cache_dir is none, then we try to download just the latest manifest for the given train, and compare it to the current system. If cache_dir is set, then we use the manifest in that directory. """ conf = Configuration.Configuration() new_manifest = None if cache_dir: try: mfile = VerifyUpdate(cache_dir) if mfile is None: return None except UpdateBusyCacheException: log.debug("Cache directory %s is busy, so no update available" % cache_dir) return None except (UpdateIncompleteCacheException, UpdateInvalidCacheException) as e: log.error( "CheckForUpdate(train = %s, cache_dir = %s): Got exception %s, removing cache" % (train, cache_dir, str(e))) RemoveUpdate(cache_dir) return None except BaseException as e: log.error( "CheckForUpdate(train=%s, cache_dir = %s): Got exception %s" % (train, cache_dir, str(e))) raise e # We always want a valid signature when doing an update new_manifest = Manifest.Manifest(require_signature=True) try: new_manifest.LoadFile(mfile) except Exception as e: log.error("Could not load manifest due to %s" % str(e)) raise e else: try: new_manifest = conf.FindLatestManifest(train=train, require_signature=True) except Exception as e: log.error("Could not find latest manifest due to %s" % str(e)) if new_manifest is None: raise UpdateManifestNotFound("Manifest could not be found!") # If new_manifest is not the requested train, then we don't have an update to do if train and train != new_manifest.Train(): log.debug( "CheckForUpdate(train = %s, cache_dir = %s): Wrong train in caache (%s)" % (train, cache_dir, new_manifest.Train())) return None diffs = Manifest.CompareManifests(conf.SystemManifest(), new_manifest) if diffs is None or len(diffs) == 0: return None if handler: for (pkg, op, old) in diffs: handler(op, pkg, old) return new_manifest