def PendingUpdates(directory):
"""
Return a list (a la CheckForUpdates handler right now) of
changes between the currently installed system and the
downloaded contents in <directory>. If <directory>'s values
are incomplete or invalid for whatever reason, return
None. "Incomplete" means a necessary file for upgrading
from the current system is not present; "Invalid" means that
one part of it is invalid -- manifest is not valid, signature isn't
valid, checksum for a file is invalid, or the stashed sequence
number does not match the current system's sequence.
"""
mani_file = None
conf = Configuration.Configuration()
try:
mani_file = VerifyUpdate(directory)
except UpdateBusyCacheException:
log.debug("Cache directory %s is busy, so no update available" %
directory)
return None
except (UpdateIncompleteCacheException, UpdateInvalidCacheException) as e:
log.error(str(e))
RemoveUpdate(directory)
return None
except BaseException as e:
log.error(
"Got exception %s while trying to determine pending updates" %
str(e))
return None
if mani_file:
new_manifest = Manifest.Manifest()
new_manifest.LoadFile(mani_file)
diffs = Manifest.CompareManifests(conf.SystemManifest(), new_manifest)
return diffs
return None
def CheckForUpdates(root=None, handler=None, train=None, cache_dir=None):
"""
Check for an updated manifest.
Very simple, uses the configuration module.
Returns the new manifest if there is an update,
and None otherwise.
(It determines if there is an update if the latest-found
manifest contains differences from the current system
manifest.)
The optional argument handler is a function that
will be called for each difference in the new manifest
(if there is one); it will be called with three
arguments: operation, package, old package.
operation will be "delete", "upgrade", or "install";
old package will be None for delete and install.
The optional cache_dir parameter indicates that it
should look in that directory first, rather than
going over the network. (Unlike the similar code
in freenas-update, this will not [at this time]
download the package files and store them in
cache_dir.)
"""
conf = Configuration.Configuration(root)
cur = conf.SystemManifest()
m = None
# Let's check cache_dir if it is set
if cache_dir and (not train or train == cur.Train()):
if os.path.exists(cache_dir + "/MANIFEST"):
# Okay, let's load it up.
m = Manifest.Manifest()
try:
m.LoadPath(cache_dir + "/MANIFEST")
if m.Train() != cur.Train():
# Should we get rid of the cache?
m = None
except:
m = None
if m is None:
m = conf.FindLatestManifest(train=train)
log.debug("Current sequence = %s, available sequence = %s" %
(cur.Sequence(), m.Sequence() if m is not None else "None"))
if m is None:
raise ValueError("Manifest could not be found!")
diffs = Manifest.CompareManifests(cur, m)
update = False
for (pkg, op, old) in diffs:
update = True
if handler is not None:
handler(op, pkg, old)
return m if update else None
def update(self, job, attrs=None):
"""
Downloads (if not already in cache) and apply an update.
"""
train = (attrs or {}).get('train') or self.get_train()
location = self.middleware.call('notifier.get_update_location')
handler = UpdateHandler(self, job)
update = Update.DownloadUpdate(
train,
location,
check_handler=handler.check_handler,
get_handler=handler.get_handler,
)
if update is False:
raise ValueError('No update available')
new_manifest = Manifest.Manifest(require_signature=True)
new_manifest.LoadPath('{}/MANIFEST'.format(location))
Update.ApplyUpdate(
location,
install_handler=handler.install_handler,
)
self.middleware.call('cache.put', 'update.applied', True)
return True
async def update(self, job, attrs=None):
"""
Downloads (if not already in cache) and apply an update.
"""
attrs = attrs or {}
train = attrs.get('train') or (
await self.middleware.call('update.get_trains'))['selected']
location = await self.middleware.call('notifier.get_update_location')
job.set_progress(0, 'Retrieving update manifest')
handler = UpdateHandler(self, job)
update = Update.DownloadUpdate(
train,
location,
check_handler=handler.check_handler,
get_handler=handler.get_handler,
)
if update is False:
raise ValueError('No update available')
new_manifest = Manifest.Manifest(require_signature=True)
new_manifest.LoadPath('{}/MANIFEST'.format(location))
Update.ApplyUpdate(
location,
install_handler=handler.install_handler,
)
await self.middleware.call('cache.put', 'update.applied', True)
if attrs.get('reboot'):
await self.middleware.call('system.reboot', {'delay': 10})
return True
def CheckForUpdates(root = None, handler = None):
"""
Check for an updated manifest.
Very simple, uses the configuration module.
Returns the new manifest if there is an update,
and None otherwise.
(It determines if there is an update if the latest-found
manifeset's sequence number is larger than the current
sequence number.)
The optional argument handler is a function that
will be called for each difference in the new manifest
(if there is one); it will be called with three
arguments: operation, package, old package.
operation will be "delete", "upgrade", or "install";
old package will be None for delete and install.
"""
conf = Configuration.Configuration(root)
cur = conf.SystemManifest()
m = conf.FindLatestManifest()
print >> sys.stderr, "Current sequence = %d, available sequence = %d" % (cur.Sequence(), m.Sequence() if m is not None else 0)
if m is not None and m.Sequence() > cur.Sequence():
if handler is not None:
diffs = Manifest.CompareManifests(cur, m)
for (pkg, op, old) in diffs:
handler(op, pkg, old)
return m
return None
def CheckForUpdates(root = None, handler = None):
"""
Check for an updated manifest.
Very simple, uses the configuration module.
Returns the new manifest if there is an update,
and None otherwise.
(It determines if there is an update if the latest-found
manifest contains differences from the current system
manifest.)
The optional argument handler is a function that
will be called for each difference in the new manifest
(if there is one); it will be called with three
arguments: operation, package, old package.
operation will be "delete", "upgrade", or "install";
old package will be None for delete and install.
"""
conf = Configuration.Configuration(root)
cur = conf.SystemManifest()
m = conf.FindLatestManifest()
log.debug("Current sequence = %s, available sequence = %s" % (cur.Sequence(), m.Sequence()
if m is not None else "None"))
if m is None:
raise ValueError("Manifest could not be found!")
diffs = Manifest.CompareManifests(cur, m)
update = False
for (pkg, op,old) in diffs:
update = True
if handler is not None:
handler(op, pkg, old)
return m if update else None
def main(handler, args):
setproctitle('updated')
handler.pid = os.getpid()
handler.dump()
if args.download:
log.debug('Starting DownloadUpdate')
Update.DownloadUpdate(
args.train,
args.cache,
check_handler=handler.get_handler,
get_handler=handler.get_file_handler,
)
log.debug('DownloadUpdate finished')
new_manifest = Manifest.Manifest(require_signature=True)
try:
new_manifest.LoadPath(args.cache + '/MANIFEST')
except ManifestInvalidSignature as e:
log.error("Cached manifest has invalid signature: %s" % str(e))
raise
update_version = new_manifest.Version()
if args.apply:
log.debug('Starting ApplyUpdate')
handler.reboot = Update.ApplyUpdate(
args.cache,
install_handler=handler.install_handler,
)
log.debug('ApplyUpdate finished')
if handler.reboot:
# Create Alert that update is applied and system should now be rebooted
create_update_alert(update_version)
def Install(root = None, manifest = None):
"""
Perform an install. Uses the system manifest, and installs
into root. root must be set.
"""
if root is None:
print >> sys.stderr, "Install must have target root specified"
usage()
conf = Configuration.Configuration()
if manifest is not None:
cur = Manifest.Manifest()
try:
cur.LoadPath(manifest)
except Exception as e:
print >> sys.stderr, "Could not load manifest from %s: %s" % (manifest, str(e))
return False
else:
try:
cur = Configuration.SystemManifest()
except:
print >> sys.stderr, "Cannot get system manifest"
return False
if cur is None or cur.Packages() is None:
raise Exception("Cannot load configuration")
print "Want to install into %s" % root
#
# To install, we have to grab each package in the manifest,
# and install them into the specified root directory.
# When we are done, we write out the system manifest into
# the manifest directory.
for pkg in cur.Packages():
print "Package %s" % pkg.Name()
filename = Manifest.FormatName(pkg.Name(), pkg.Version())
f = conf.FindPackage(filename, pkg.Checksum())
if f is None:
print >> sys.stderr, "\tCould not find package file for %s" % filename
return False
else:
if Installer.install_file(f, root) == False:
print >> sys.stderr, "Could not install package %s" % filename
return False
else:
print "%s installed" % pkg.Name()
conf.StoreManifest(cur, root)
return True
def install_impl(self, job, location):
old_manifest = Configuration.Configuration().SystemManifest()
new_manifest = Manifest.Manifest(require_signature=True)
new_manifest.LoadPath('{}/MANIFEST'.format(location))
old_version = old_manifest.Version()
new_version = new_manifest.Version()
if not can_update(old_version, new_version):
raise CallError(f'Unable to downgrade from {old_version} to {new_version}')
return self.middleware.call_sync('update.install_impl_job', job.id, location).wait_sync(raise_error=True)
def main():
config_file = None
mani_file = None
try:
opts, args = getopt.getopt(sys.argv[1:], "C:M:")
except getopt.GetoptError as err:
print str(err)
usage()
for o, a in opts:
if o == "-C":
config = a
elif o == "-M":
mani_file = a
else:
usage()
if len(args) == 0:
usage()
conf = Configuration.Configuration(file=config_file, nopkgdb=True)
if mani_file is not None:
mani = Manifest.Manifest(conf)
mani.LoadPath(mani_file)
else:
mani = conf.SystemManifest()
if mani is None:
print >> sys.stderr, "Unable to load manifest"
return (1)
if args[0] == "list":
list_cmd(mani, args[1:])
elif args[0] == "train":
print mani.Train()
elif args[0] == "sequence":
print mani.Sequence()
elif args[0] == "version":
if mani.Version() is not None:
print mani.Version()
elif args[0] == "notes":
if mani.Notes() is not None:
print mani.Notes()
elif args[0] == "show":
show_cmd(mani, args[1:])
elif args[0] == "sign":
sign_manifest(mani, args[1:])
else:
usage()
return 0
def PendingUpdatesChanges(directory):
"""
Return a list (a la CheckForUpdates handler right now) of
changes between the currently installed system and the
downloaded contents in <directory>. If <directory>'s values
are incomplete or invalid for whatever reason, return
None. "Incomplete" means a necessary file for upgrading
from the current system is not present; "Invalid" means that
one part of it is invalid -- manifest is not valid, signature isn't
valid, checksum for a file is invalid, or the stashed sequence
number does not match the current system's sequence.
"""
mani_file = None
conf = Configuration.Configuration()
try:
mani_file = VerifyUpdate(directory)
except UpdateBusyCacheException:
log.debug("Cache directory %s is busy, so no update available" %
directory)
raise
except (UpdateIncompleteCacheException, UpdateInvalidCacheException) as e:
log.error(str(e))
RemoveUpdate(directory)
raise
except BaseException as e:
log.error(
"Got exception %s while trying to determine pending updates" %
str(e))
raise
if mani_file:
new_manifest = Manifest.Manifest(require_signature=True)
try:
new_manifest.LoadFile(mani_file)
except ManifestInvalidSignature as e:
log.error("Invalid signature in cached manifest: %s" % str(e))
raise
# This returns a set of differences.
# But we shouldn't rely on it until we can look at what we've
# actually downloaded. To do that, we need to look at any
# package differences (diffs["Packages"]), and check the
# updates if that's what got downloaded.
# By definition, if there are no Packages differences, a reboot
# isn't required.
diffs = GetUpdateChanges(conf.SystemManifest(),
new_manifest,
cache_dir=directory)
return diffs
else:
return None
async def update(self, job, attrs=None):
"""
Downloads (if not already in cache) and apply an update.
"""
attrs = attrs or {}
trains = await self.middleware.call('update.get_trains')
train = attrs.get('train') or trains['selected']
if attrs.get('train'):
await self.middleware.run_in_thread(self.__set_train, attrs.get('train'), trains)
location = await self.middleware.call('update.get_update_location')
job.set_progress(0, 'Retrieving update manifest')
handler = UpdateHandler(self, job)
update = Update.DownloadUpdate(
train,
location,
check_handler=handler.check_handler,
get_handler=handler.get_handler,
)
if update is False:
raise ValueError('No update available')
new_manifest = Manifest.Manifest(require_signature=True)
new_manifest.LoadPath('{}/MANIFEST'.format(location))
Update.ApplyUpdate(
location,
install_handler=handler.install_handler,
)
await self.middleware.call('cache.put', 'update.applied', True)
if (
await self.middleware.call('system.is_freenas') or
(
await self.middleware.call('failover.licensed') and
await self.middleware.call('failover.status') != 'BACKUP'
)
):
await self.middleware.call('update.take_systemdataset_samba4_snapshot')
if attrs.get('reboot'):
await self.middleware.call('system.reboot', {'delay': 10})
return True
outfile = a
else:
usage()
pkgs = args
if sequence is None:
# Use time
sequence = int(time.time())
if (trainname is None) or len(pkgs) == 0:
usage()
# We need a configuration to do searching
conf = Configuration.Configuration(file=config_file)
mani = Manifest.Manifest(conf)
mani.SetTrain(trainname)
mani.SetSequence(sequence)
if timestamp:
mani.SetTimeStamp(timestamp)
if releasename is not None:
mani.SetVersion(releasename)
if notesfile is not None:
notes = []
with open(notesfile, "r") as f:
for line in f:
notes.append(line)
mani.SetNotes(notes)
if package_dir is not None:
else: usage()
if len(args) != 1:
usage()
root = args[0]
config = Configuration.Configuration()
if package_dir is not None:
config.SetPackageDir(package_dir)
if mani_file is None:
manifest = config.SystemManifest()
else:
# We ignore the signature because freenas-install is
# called from the ISO install, and the GUI install, which
# have their own checksums elsewhere.
manifest = Manifest.Manifest(config, ignore_signature=True)
manifest.LoadPath(mani_file)
installer = Installer.Installer(manifest=manifest,
root=root,
config=config)
if installer.GetPackages() != True:
print >> sys.stderr, "Huh, could not install and yet it returned"
installer.InstallPackages(PrintProgress)
manifest.Save(root)
sys.exit(0)
len(possible_disks)))
raise InstallationError("Insufficient number of disks found")
install_config['disks'] = possible_disks
if install_config["upgrade"]:
LogIt("Currently cannot handle an upgrade, sorry")
raise InstallationError("Cannot currently handle an upgrade")
install_disks = []
for disk in install_config["disks"]:
install_disks.append(Utils.Disk(disk))
# Okay, at this point we are nearly all set. Let's set up some variables
manifest_path = os.path.join(args.buildroot[0],
"release/LATEST/{}-MANIFEST".format(Project()))
manifest = Manifest.Manifest()
manifest.LoadPath(manifest_path)
package_dir = os.path.join(args.buildroot[0], "release/LATEST/Packages")
fn_conf = Configuration.SystemConfiguration()
template_dir = os.path.join(args.buildroot[0], "objs/instufs/data")
try:
Install(interactive=False,
manifest=manifest,
config=fn_conf,
package_directory=package_dir,
disks=install_disks,
efi=install_config.get("format", "bios") == "efi",
def do_install():
"""
Do the UI for the install.
This will either return, or raise an exception. DialogEscape means that
the installer should start over.
If we have any command-line arguments, let's handle them now
"""
arg_parser = argparse.ArgumentParser(description=Title(), prog="Installer")
arg_parser.register('type', 'bool',
lambda x: x.lower() in ["yes", "y", "true", "t"])
arg_parser.add_argument('-p',
'--project',
dest='project',
default=Project(),
help="Specify project (default {})".format(
Project()))
arg_parser.add_argument('-T',
'--train',
dest="train",
help="Specify train name")
arg_parser.add_argument('-U',
'--url',
dest="url",
help="URL to use when fetching files")
arg_parser.add_argument('-D',
"--data",
dest='data_dir',
help='Path to /data prototype directory')
arg_parser.add_argument('-M',
'--manifest',
dest='manifest',
help="Path to manifest file")
arg_parser.add_argument('-P',
'--packages',
dest='package_dir',
help='Path to package files')
arg_parser.add_argument(
"-B",
"--trampoline",
dest='trampoline',
default=True,
type='bool',
help="Run post-install scripts on reboot (default)")
args = arg_parser.parse_args()
if args:
LogIt("Command line args: {}".format(args))
SetProject(args.project)
try:
validate_system()
except ValidationError as e:
LogIt("Could not validate system: {}".format(e.message))
Dialog.MessageBox(
Title(),
"\nSystem memory is too small. Minimum memory size is 8Gbytes",
height=10,
width=45).run()
return
if args.manifest:
if os.path.exists(args.manifest):
manifest_path = args.manifest
else:
Dialog.MessageBox(
Title(),
"A manifest file was specified on the command line, but does not exist. The manifest file specified was\n\n\t{}"
.format(args.manifest),
height=15,
width=45).run()
raise InstallationError(
"Command-line manifest file {} does not exist".foramt(
args.manifet))
else:
manifest_path = "/.mount/{}-MANIFEST".format(Project())
if not os.path.exists(manifest_path):
manifest_path = None
package_dir = args.package_dir or "/.mount/{}/Packages".format(Project())
if not os.path.exists(package_dir):
# This will be used later to see if we should try downloading
package_dir = None
# If we aren't given a URL, we can try to use the default url.
# If we have a manifest file, we can figure out train;
# if we don't have a train or manifest file, we're not okay.
if (not manifest_path and not args.train):
LogIt("Command-line URL {}, train {}, manifest {}".format(
args.url, args.train, manifest_path))
box = Dialog.MessageBox(Title(), "", height=15, width=45)
box.text = "Neither a manifest file nor train were specified"
box.run()
raise InstallationError("Incorrect command-line arguments given")
conf = Configuration.SystemConfiguration()
if conf is None:
raise RuntimeError("No configuration?!")
# Okay, if we're going to have to download, let's make an update server object
if args.url:
temp_update_server = Configuration.UpdateServer(
name="Installer Server", url=args.url, signing=False)
# This is SO cheating
# It can't write to the file, but it does that after setting it.
try:
conf.AddUpdateServer(temp_update_server)
except:
pass
conf.SetUpdateServer("Installer Server", save=False)
# If we have a train, and no manifest file, let's grab one
if manifest_path:
manifest = Manifest.Manifest()
try:
manifest.LoadPath(manifest_path)
except:
manifest = None
else:
manifest = None
if args.train and not manifest:
try:
status = Dialog.MessageBox(
Title(),
"Attempting to download manifest for train {}".format(
args.train),
height=15,
width=30,
wait=False)
status.clear()
status.run()
except:
pass
try:
manifest = conf.FindLatestManifest(train=args.train,
require_signature=False)
except:
manifest = None
# At this point, if we don't have a manifest, we can't do anything
if manifest is None:
LogIt("Could not load a manifest")
text = "Despite valiant efforts, no manifest file could be located."
Dialog.MessageBox(Title(), text, height=15, width=30).run()
raise InstallationError("Unable to locate a manifest file")
LogIt("Manifest: Version {}, Train {}, Sequence {}".format(
manifest.Version(), manifest.Train(), manifest.Sequence()))
do_upgrade = False
boot_method = None
disks = SelectDisks()
if not disks:
try:
Dialog.MessageBox(Title(),
"No suitable disks were found for installation",
height=15,
width=60).run()
except:
pass
raise InstallationError("No disks selected for installation")
if found_bootpool:
if UpgradePossible():
text = """The {} installer can upgrade the existing {} installation.
Do you want to upgrade?""".format(Project(), Project())
yesno = Dialog.YesNo("Perform Upgrade",
text,
height=12,
width=60,
yes_label="Upgrade",
no_label="Do not Upgrade",
default=True)
do_upgrade = yesno.result
else:
Dialog.MessageBox("No upgrade possible", "").run()
format_disks = True
if found_bootpool:
# If the selected disks are not the same as the existing boot-pool
# disks, then we _will_ be formatting, and do not ask this question.
disk_set = set([x.name for x in disks])
pool_set = set([Utils.Disk(x).name for x in found_bootpool.disks])
LogIt("disk_set = {}, pool_set = {}".format(disk_set, pool_set))
if pool_set == disk_set:
yesno = Dialog.YesNo(
Title(),
"The {} installer can reformat the disk{}, or create a new boot environment.\nReformatting will erase all of your data"
.format(Project(), "s" if len(disks) > 1 else ""),
height=10,
width=60,
yes_label="Re-format",
no_label="Create New BE",
default=True)
format_disks = yesno.result
yesno.clear()
if format_disks:
# If there is already a freenas-boot, and we're not using all of
# the disks in it, then this will cause problems.
# If we made it this far, there is only one freenas-boot pool.
if found_bootpool:
pool_disks = [Utils.Disk(x) for x in found_bootpool.disks]
disk_set = set([x.name for x in disks])
pool_set = set([x.name for x in pool_disks])
LogIt("disk_set = {}, pool_set = {}".format(disk_set, pool_set))
if not pool_set <= disk_set:
# This means there would be two freenas-boot pools, which
# is too much of a problem.
yesno = Dialog.YesNo(
Title(),
"The existing boot pool contains disks that are not in the selected set of disks, which would result in errors. Select Start Over, or press Escape, otherwise the {} installer will destroy the existing pool"
.format(Project()),
width=60,
yes_label="Destroy Pool",
no_label="Start over",
default=False)
yesno.prompt += "\nSelected Disks: " + " ,".join(
sorted([x.name for x in disks]))
yesno.prompt += "\nPool Disks: " + " ,".join(
sorted([x.name for x in pool_disks]))
if yesno.result is False:
raise Dialog.DialogEscape
current_method = BootMethod()
yesno = Dialog.YesNo(
"Boot Method",
"{} can boot via BIOS or (U)EFI. Selecting the wrong method can result in a non-bootable system"
.format(Project()),
height=10,
width=60,
yes_label="BIOS",
no_label="(U)EFI",
default=False if current_method == "efi" else True)
if yesno.result is True:
boot_method = "bios"
else:
boot_method = "efi"
if not do_upgrade:
# Ask for root password
while True:
password_fields = [
Dialog.FormItem(
Dialog.FormLabel("Password:"******"",
width=20,
maximum_input=50,
hidden=True)),
Dialog.FormItem(
Dialog.FormLabel("Confirm Password:"******"",
width=20,
maximum_input=50,
hidden=True)),
]
try:
password_input = Dialog.Form(
"Root Password",
"Enter the root password. (Escape to quit, or select No Password)",
width=60,
height=15,
cancel_label="No Password",
form_height=10,
form_items=password_fields)
results = password_input.result
if results and results[0].value.value != results[1].value.value:
Dialog.MessageBox("Password Error",
"Passwords did not match",
width=35,
ok_label="Try again").run()
continue
else:
new_password = results[0].value.value if results else None
break
except Dialog.DialogEscape:
try:
Diallog.MessageBox("No Password Selected",
"You have selected an empty password",
height=7,
width=35).run()
except:
pass
new_password = None
break
# I'm not sure if this should be done here, or in Install()
if package_dir is None:
cache_dir = tempfile.mkdtemp()
else:
cache_dir = package_dir
try:
Utils.GetPackages(manifest, conf, cache_dir, interactive=True)
except BaseException as e:
LogIt("GetPackages raised an exception {}".format(str(e)))
if package_dir is None:
shutil.rmtree(cache_dir, ignore_errors=True)
raise
LogIt("Done getting packages?")
# Let's confirm everything
text = "The {} Installer will perform the following actions:\n\n".format(
Project())
height = 10
if format_disks:
text += "* The following disks will be reformatted, and all data lost:\n"
height += 1
for disk in disks:
text += "\t* {} {} ({}bytes)\n".format(disk.name,
disk.description[:25],
SmartSize(disk.size))
height += 1
if found_bootpool:
text += "* The existing boot pool will be destroyed\n"
height += 1
text += "* {} Booting\n".format(
"BIOS" if boot_method is "bios" else "(U)EFI")
else:
text += "* A new Boot Environment will be created\n"
height += 1
if do_upgrade:
text += "* {} will be upgraded\n".format(Project())
else:
text += "* {} will be freshly installed\n".format(Project())
height += 1
yesno.prompt = text
yesno.default = False
yesno = Dialog.YesNo("{} Installation Confirmation".format(Project()),
text,
height=min(15, height),
width=60,
default=False)
if yesno.result == False:
LogIt("Installation aborted at first confirmation")
raise Dialog.DialogEscape
if format_disks:
yesno = Dialog.YesNo(
"LAST CHANCE",
"The {} installer will format the selected disks and all data on them will be erased.\n\nSelect Start Over or hit Escape to start over!"
.format(Project()),
height=10,
width=50,
yes_label="Continue",
no_label="Start Over",
default=False)
if yesno.result is False:
LogIt("Installation aborted at second confirmation")
raise Dialog.DialogEscape
# This may take a while, it turns out
try:
status = Dialog.MessageBox(Title(),
"\nBeginning installation",
height=7,
width=25,
wait=False)
status.clear()
status.run()
except:
pass
# Okay, time to do the install
with InstallationHandler() as handler:
try:
Install.Install(
interactive=True,
manifest=manifest,
config=conf,
package_directory=cache_dir,
disks=disks if format_disks else None,
efi=True if boot_method is "efi" else False,
upgrade_from=found_bootpool if found_bootpool else None,
upgrade=do_upgrade,
data_dir=args.data_dir if args.data_dir else "/data",
package_handler=handler.start_package,
progress_handler=handler.package_update,
password=None if do_upgrade else new_password,
trampoline=args.trampoline)
except BaseException as e:
LogIt("Install got exception {}".format(str(e)))
raise
return
def GetUpdateChanges(old_manifest, new_manifest, cache_dir=None):
"""
This is used by both PendingUpdatesChanges() and CheckForUpdates().
The difference between the two is that the latter doesn't necessarily
have a cache directory, so if cache_dir is none, we have to assume the
update package exists.
This returns a dictionary that will have at least "Reboot" as a key.
"""
def MergeServiceList(base_list, new_list):
"""
Merge new_list into base_list.
For each service in new_list (which is a dictionary),
if the value is True, add it to base_list.
If new_list is an array, simply add each item to
base_list; if it's a dict, we check the value.
"""
if new_list is None:
return base_list
if isinstance(new_list, list):
for svc in new_list:
if not svc in base_list:
base_list.append(svc)
elif isinstance(new_list, dict):
for svc, val in new_list.iteritems():
if val:
if not svc in base_list:
base_list.append(svc)
return base_list
svcs = []
diffs = Manifest.DiffManifests(old_manifest, new_manifest)
if len(diffs) == 0:
return None
reboot = False
if REQUIRE_REBOOT:
reboot = True
if "Packages" in diffs:
# Look through the install/upgrade packages
for pkg, op, old in diffs["Packages"]:
if op == "delete":
continue
if op == "install":
if pkg.RequiresReboot() == True:
reboot = True
else:
pkg_services = pkg.RestartServices()
if pkg_services:
svcs = MergeServiceList(svcs, pkg_services)
elif op == "upgrade":
# A bit trickier.
# If there is a list of services to restart, the update
# path wants to look at that rather than the requires reboot.
# However, one service could be to reboot (this is handled
# below).
upd = pkg.Update(old.Version())
if cache_dir:
update_fname = os.path.join(cache_dir,
pkg.FileName(old.Version()))
else:
update_fname = None
if upd and (update_fname is None
or os.path.exists(update_fname)):
pkg_services = upd.RestartServices()
if pkg_services:
svcs = MergeServiceList(svcs, pkg_services)
else:
if upd.RequiresReboot() == True:
reboot = True
else:
# Have to assume the full package exists
if pkg.RequiresReboot() == True:
reboot = True
else:
pkg_services = pkg.RestartServices()
if pkg_services:
svcs = MergeServiceList(svcs, pkg_services)
else:
reboot = False
if len(diffs) == 0:
return None
if not reboot and svcs:
if not VerifyServices(svcs):
reboot = True
else:
diffs["Restart"] = svcs
diffs["Reboot"] = reboot
return diffs
def run(self, reboot_post_install=False):
self.message = 'Applying Updates...'
self.set_progress(0)
handler = UpdateHandler(self.dispatcher,
update_progress=self.update_progress)
cache_dir = self.dispatcher.call_sync('update.update_cache_getter',
'cache_dir')
if cache_dir is None:
try:
cache_dir = self.dispatcher.call_sync(
'system_dataset.request_directory', 'update')
except RpcException:
cache_dir = '/var/tmp/update'
new_manifest = Manifest.Manifest(require_signature=True)
try:
new_manifest.LoadPath(cache_dir + '/MANIFEST')
except ManifestInvalidSignature as e:
logger.error("Cached manifest has invalid signature: %s" % str(e))
raise TaskException(errno.EINVAL, str(e))
except FileNotFoundError as e:
raise TaskException(
errno.EIO,
'No Manifest file found at path: {0}'.format(cache_dir +
'/MANIFEST'))
version = new_manifest.Version()
# Note: for now we force reboots always, TODO: Fix in M3-M4
try:
result = ApplyUpdate(cache_dir,
install_handler=handler.install_handler,
force_reboot=True)
except ManifestInvalidSignature as e:
logger.debug(
'UpdateApplyTask Error: Cached manifest has invalid signature: %s',
e)
raise TaskException(
errno.EINVAL,
'Cached manifest has invalid signature: {0}'.format(str(e)))
except UpdateBootEnvironmentException as e:
logger.debug('UpdateApplyTask Boot Environment Error: {0}'.format(
str(e)))
raise TaskException(errno.EAGAIN, str(e))
except UpdatePackageException as e:
logger.debug('UpdateApplyTask Package Error: {0}'.format(str(e)))
raise TaskException(errno.EAGAIN, str(e))
except Exception as e:
raise TaskException(
errno.EAGAIN,
'Got exception {0} while trying to Apply Updates'.format(
str(e)))
if result is None:
raise TaskException(errno.ENOENT,
'No downloaded Updates available to apply.')
handler.finished = True
handler.emit_update_details()
self.dispatcher.call_sync('update.update_alert_set', 'UpdateInstalled',
version)
update_cache_value_dict = default_update_dict.copy()
update_cache_value_dict.update({
'installed': True,
'installed_version': version
})
self.dispatcher.call_sync('update.update_cache_putter',
update_cache_value_dict)
self.message = "Updates Finished Installing Successfully"
if reboot_post_install:
self.message = "Scheduling user specified reboot post succesfull update"
# Note not using subtasks on purpose as they do not have queuing logic
self.dispatcher.submit_task('system.reboot', 3)
self.set_progress(100)
def DownloadUpdate(train, directory, get_handler=None, check_handler=None):
"""
Download, if necessary, the LATEST update for train; download
delta packages if possible. Checks to see if the existing content
is the right version. In addition to the current caching code, it
will also stash the current sequence when it downloads; this will
allow it to determine if a reboot into a different boot environment
has happened. This will remove the existing content if it decides
it has to redownload for any reason.
"""
import shutil
import fcntl
conf = Configuration.Configuration()
mani = conf.SystemManifest()
# First thing, let's get the latest manifest
latest_mani = conf.FindLatestManifest(train)
if latest_mani is None:
# That really shouldnt happen
log.error("Unable to find latest manifest for train %s" % train)
return False
cache_mani = Manifest.Manifest()
try:
mani_file = VerifyUpdate(directory)
if mani_file:
cache_mani.LoadFile(mani_file)
if cache_mani.Sequence() == latest_mani.Sequence():
# Woohoo!
mani_file.close()
return True
# Not the latest
mani_file.close()
RemoveUpdate(directory)
mani_file = None
except UpdateBusyCacheException:
log.debug("Cache directory %s is busy, so no update available" %
directory)
return False
except (UpdateIncompleteCacheException, UpdateInvalidCacheException) as e:
# It's incomplete, so we need to remove it
log.error("DownloadUpdate(%s, %s): Got exception %s; removing cache" %
(train, directory, str(e)))
RemoveUpdate(directory)
except BaseException as e:
log.error("Got exception %s while trying to prepare update cache" %
str(e))
raise e
# If we're here, then we don't have a (valid) cached update.
RemoveUpdate(directory)
try:
os.makedirs(directory)
except BaseException as e:
log.error("Unable to create directory %s: %s" % (directory, str(e)))
return False
try:
mani_file = open(directory + "/MANIFEST", "wxb")
except (IOError, Exception) as e:
log.error("Unale to create manifest file in directory %s" %
(directory, str(e)))
return False
try:
fcntl.lockf(mani_file, fcntl.LOCK_EX | fcntl.LOCK_NB, 0, 0)
except (IOError, Exception) as e:
log.debug("Unable to lock manifest file: %s" % str(e))
mani_file.close()
return False
# Find out what differences there are
diffs = Manifest.CompareManifests(mani, latest_mani)
download_packages = []
for pkg, op, old in diffs:
if op == "delete":
continue
download_packages.append(pkg)
# Next steps: download the package files.
for indx, pkg in enumerate(download_packages):
if check_handler:
check_handler(indx + 1, pkg=pkg, pkgList=download_packages)
pkg_file = conf.FindPackageFile(pkg,
save_dir=directory,
handler=get_handler)
if pkg_file is None:
log.error("Could not download package file for %s" % pkg.Name())
RemoveUpdate(directory)
return False
# Then save the manifest file.
latest_mani.StoreFile(mani_file)
# Create the SEQUENCE file.
with open(directory + "/SEQUENCE", "w") as f:
f.write("%s" % conf.SystemManifest().Sequence())
# Then return True!
mani_file.close()
return True
def CheckForUpdates(handler=None,
train=None,
cache_dir=None,
diff_handler=None):
"""
Check for an updated manifest. If cache_dir is none, then we try
to download just the latest manifest for the given train, and
compare it to the current system. If cache_dir is set, then we
use the manifest in that directory.
"""
conf = Configuration.Configuration()
new_manifest = None
if cache_dir:
try:
mfile = VerifyUpdate(cache_dir)
if mfile is None:
return None
except UpdateBusyCacheException:
log.debug("Cache directory %s is busy, so no update available" %
cache_dir)
return None
except (UpdateIncompleteCacheException,
UpdateInvalidCacheException) as e:
log.error(
"CheckForUpdate(train = %s, cache_dir = %s): Got exception %s, removing cache"
% (train, cache_dir, str(e)))
RemoveUpdate(cache_dir)
return None
except BaseException as e:
log.error(
"CheckForUpdate(train=%s, cache_dir = %s): Got exception %s" %
(train, cache_dir, str(e)))
raise e
# We always want a valid signature when doing an update
new_manifest = Manifest.Manifest(require_signature=True)
try:
new_manifest.LoadFile(mfile)
except Exception as e:
log.error("Could not load manifest due to %s" % str(e))
raise e
else:
try:
new_manifest = conf.FindLatestManifest(train=train,
require_signature=True)
except Exception as e:
log.error("Could not find latest manifest due to %s" % str(e))
if new_manifest is None:
raise UpdateManifestNotFound("Manifest could not be found!")
# If new_manifest is not the requested train, then we don't have an update to do
if train and train != new_manifest.Train():
log.debug(
"CheckForUpdate(train = %s, cache_dir = %s): Wrong train in caache (%s)"
% (train, cache_dir, new_manifest.Train()))
return None
diffs = GetUpdateChanges(conf.SystemManifest(), new_manifest)
if diffs is None or len(diffs) == 0:
return None
log.debug("CheckForUpdate: diffs = %s" % diffs)
if "Packages" in diffs:
for (pkg, op, old) in diffs["Packages"]:
if handler:
handler(op, pkg, old)
if diff_handler:
diff_handler(diffs)
return new_manifest
def ApplyUpdate(directory, install_handler=None):
"""
Apply the update in <directory>. As with PendingUpdates(), it will
have to verify the contents before it actually installs them, so
it has the same behaviour with incomplete or invalid content.
"""
rv = False
conf = Configuration.Configuration()
changes = PendingUpdates(directory)
if changes is None:
# That could have happened for multiple reasons.
# PendingUpdates should probably throw an exception
# on error
return False
# Do I have to worry about a race condition here?
new_manifest = Manifest.Manifest()
new_manifest.LoadPath(directory + "/MANIFEST")
conf.SetPackageDir(directory)
deleted_packages = []
updated_packages = []
for (pkg, op, old) in changes:
if op == "delete":
log.debug("Delete package %s" % pkg.Name())
deleted_packages.append(pkg)
elif op == "install":
log.debug("Install package %s" % pkg.Name())
else:
log.debug("Upgrade package %s-%s to %s-%s" %
(old.Name(), old.Version(), pkg.Name(), pkg.Version()))
updated_packages.append(pkg)
if len(deleted_packages) == 0 and len(updated_packages) == 0:
# The manifest may have other differences, so we should
# probably do something.
log.debug("New manifest has no package changes, what should we do?")
RemoveUpdate(directory)
return True
# Now we start doing the update!
clone_name = None
mount_point = None
try:
clone_name = "%s-%s" % (Avatar(), new_manifest.Sequence())
if CreateClone(clone_name) is False:
s = "Unable to create boot-environment %s" % clone_name
log.error(s)
raise Exception(s)
mount_point = MountClone(clone_name)
if mount_point is None:
s = "Unable to mount boot-environment %s" % clone_name
log.error(s)
DeleteClone(clone_name)
raise Exception(s)
# Remove any deleted packages
for pkg in deleted_packages:
log.debug("About to delete package %s" % pkg.Name())
if conf.PackageDB(mount_point).RemovePackageContents(pkg) == False:
s = "Unable to remove contents for packate %s" % pkg.Name()
if mount_point:
UnmountClone(clone_name, mount_point)
mount_point = None
DeleteClone(clone_name)
raise Exception(s)
conf.PackageDB(mount_point).RemovePackage(pkg.Name())
installer = Installer.Installer(manifest=new_manifest,
root=mount_point,
config=conf)
installer.GetPackages(pkgList=updated_packages)
log.debug("Installer got packages %s" % installer._packages)
# Now to start installing them
rv = False
if installer.InstallPackages(handler=install_handler) is False:
log.error("Unable to install packages")
raise Exception("Unable to install packages")
else:
new_manifest.Save(mount_point)
if mount_point:
if UnmountClone(clone_name, mount_point) is False:
s = "Unable to unmount clone environment %s from mount point %s" % (
clone_name, mount_point)
log.error(s)
raise Exception(s)
mount_point = None
if ActivateClone(clone_name) is False:
s = "Unable to activate clone environment %s" % clone_name
log.error(s)
raise Exception(s)
RemoveUpdate(directory)
rv = True
RunCommand("/sbin/zpool", ["scrub", "freenas-boot"])
except BaseException as e:
log.error("Update got exception during update: %s" % str(e))
if mount_point:
UnmountClone(clone_name, mount_point)
if clone_name:
DeleteClone(clone_name)
raise e
return rv
def PendingUpdatesChanges(directory):
"""
Return a list (a la CheckForUpdates handler right now) of
changes between the currently installed system and the
downloaded contents in <directory>. If <directory>'s values
are incomplete or invalid for whatever reason, return
None. "Incomplete" means a necessary file for upgrading
from the current system is not present; "Invalid" means that
one part of it is invalid -- manifest is not valid, signature isn't
valid, checksum for a file is invalid, or the stashed sequence
number does not match the current system's sequence.
"""
mani_file = None
conf = Configuration.Configuration()
try:
mani_file = VerifyUpdate(directory)
except UpdateBusyCacheException:
log.debug("Cache directory %s is busy, so no update available" %
directory)
raise
except (UpdateIncompleteCacheException, UpdateInvalidCacheException) as e:
log.error(str(e))
RemoveUpdate(directory)
raise
except BaseException as e:
log.error(
"Got exception %s while trying to determine pending updates" %
str(e))
raise
if mani_file:
new_manifest = Manifest.Manifest(require_signature=True)
try:
new_manifest.LoadFile(mani_file)
except ManifestInvalidSignature as e:
log.error("Invalid signature in cached manifest: %s" % str(e))
raise
# This returns a set of differences.
# But we shouldn't rely on it until we can look at what we've
# actually downloaded. To do that, we need to look at any
# package differences (diffs["Packages"]), and check the
# updates if that's what got downloaded.
# By definition, if there are no Packages differences, a reboot
# isn't required.
diffs = Manifest.DiffManifests(conf.SystemManifest(), new_manifest)
if len(diffs) == 0:
return None
if "Packages" in diffs:
reboot = False
# Look through the install/upgrade packages
for pkg, op, old in diffs["Packages"]:
if op == "delete":
continue
if op == "install":
if pkg.RequiresReboot() == True:
reboot = True
if op == "upgrade":
# A bit trickier.
upd = pkg.Update(old.Version())
update_fname = os.path.join(directory,
pkg.FileName(old.Version()))
if upd and os.path.exists(update_fname):
if upd.RequiresReboot() == True:
reboot = True
else:
# Have to assume the full package exists
if pkg.RequiresReboot() == True:
reboot = True
else:
reboot = False
if len(diffs) == 0:
return None
diffs["Reboot"] = reboot
return diffs
def DownloadUpdate(train, directory, get_handler=None, check_handler=None):
"""
Download, if necessary, the LATEST update for train; download
delta packages if possible. Checks to see if the existing content
is the right version. In addition to the current caching code, it
will also stash the current sequence when it downloads; this will
allow it to determine if a reboot into a different boot environment
has happened. This will remove the existing content if it decides
it has to redownload for any reason.
"""
import shutil
import fcntl
conf = Configuration.Configuration()
mani = conf.SystemManifest()
# First thing, let's get the latest manifest
try:
latest_mani = conf.FindLatestManifest(train, require_signature=True)
except ManifestInvalidSignature as e:
log.error("Latest manifest has invalid signature: %s" % str(e))
return False
if latest_mani is None:
# This probably means we have no network. Which means we have
# to trust what we've already downloaded, if anything.
log.error("Unable to find latest manifest for train %s" % train)
try:
VerifyUpdate(directory)
log.debug("Possibly with no network, cached update looks good")
return True
except:
log.debug(
"Possibly with no network, either no cached update or it is bad"
)
return False
cache_mani = Manifest.Manifest(require_signature=True)
try:
mani_file = VerifyUpdate(directory)
if mani_file:
cache_mani.LoadFile(mani_file)
if cache_mani.Sequence() == latest_mani.Sequence():
# Woohoo!
mani_file.close()
log.debug(
"DownloadUpdate: Cache directory has latest manifest")
return True
# Not the latest
mani_file.close()
mani_file = None
except UpdateBusyCacheException:
log.debug("Cache directory %s is busy, so no update available" %
directory)
return False
except (UpdateIncompleteCacheException, UpdateInvalidCacheException,
ManifestInvalidSignature) as e:
# It's incomplete, so we need to remove it
log.error("DownloadUpdate(%s, %s): Got exception %s; removing cache" %
(train, directory, str(e)))
except BaseException as e:
log.error("Got exception %s while trying to prepare update cache" %
str(e))
raise e
# If we're here, then we don't have a (valid) cached update.
log.debug("Removing invalid or incomplete cached update")
RemoveUpdate(directory)
try:
os.makedirs(directory)
except BaseException as e:
log.error("Unable to create directory %s: %s" % (directory, str(e)))
return False
try:
mani_file = open(directory + "/MANIFEST", "wxb")
except (IOError, Exception) as e:
log.error("Unale to create manifest file in directory %s" %
(directory, str(e)))
return False
try:
fcntl.lockf(mani_file, fcntl.LOCK_EX | fcntl.LOCK_NB, 0, 0)
except (IOError, Exception) as e:
log.debug("Unable to lock manifest file: %s" % str(e))
mani_file.close()
return False
# Find out what differences there are
diffs = Manifest.DiffManifests(mani, latest_mani)
if diffs is None or len(diffs) == 0:
log.debug("DownloadUpdate: No update available")
# Remove the cache directory and empty manifest file
RemoveUpdate(directory)
return False
log.debug("DownloadUpdate: diffs = %s" % diffs)
download_packages = []
reboot_required = True
if "Reboot" in diffs:
reboot_required = diffs["Reboot"]
if "Packages" in diffs:
for pkg, op, old in diffs["Packages"]:
if op == "delete":
continue
log.debug("DownloadUpdate: Will %s package %s" % (op, pkg.Name()))
download_packages.append(pkg)
log.debug("Update does%s seem to require a reboot" %
"" if reboot_required else " not")
# Next steps: download the package files.
for indx, pkg in enumerate(download_packages):
# This is where we find out for real if a reboot is required.
# To do that, we may need to know which update was downloaded.
if check_handler:
check_handler(indx + 1, pkg=pkg, pkgList=download_packages)
pkg_file = conf.FindPackageFile(pkg,
save_dir=directory,
handler=get_handler)
if pkg_file is None:
log.error("Could not download package file for %s" % pkg.Name())
RemoveUpdate(directory)
return False
# Almost done: get a changelog if one exists for the train
# If we can't get it, we don't care.
conf.GetChangeLog(train, save_dir=directory, handler=get_handler)
# Then save the manifest file.
latest_mani.StoreFile(mani_file)
# Create the SEQUENCE file.
with open(directory + "/SEQUENCE", "w") as f:
f.write("%s" % conf.SystemManifest().Sequence())
# And create the SERVER file.
with open(directory + "/SERVER", "w") as f:
f.write("%s" % conf.UpdateServerName())
# Then return True!
mani_file.close()
return True
def CheckForUpdates(handler=None,
train=None,
cache_dir=None,
diff_handler=None):
"""
Check for an updated manifest. If cache_dir is none, then we try
to download just the latest manifest for the given train, and
compare it to the current system. If cache_dir is set, then we
use the manifest in that directory.
"""
conf = Configuration.Configuration()
new_manifest = None
if cache_dir:
try:
mfile = VerifyUpdate(cache_dir)
if mfile is None:
return None
except UpdateBusyCacheException:
log.debug("Cache directory %s is busy, so no update available" %
cache_dir)
return None
except (UpdateIncompleteCacheException,
UpdateInvalidCacheException) as e:
log.error(
"CheckForUpdate(train = %s, cache_dir = %s): Got exception %s, removing cache"
% (train, cache_dir, str(e)))
RemoveUpdate(cache_dir)
return None
except BaseException as e:
log.error(
"CheckForUpdate(train=%s, cache_dir = %s): Got exception %s" %
(train, cache_dir, str(e)))
raise e
# We always want a valid signature when doing an update
new_manifest = Manifest.Manifest(require_signature=True)
try:
new_manifest.LoadFile(mfile)
except Exception as e:
log.error("Could not load manifest due to %s" % str(e))
raise e
else:
try:
new_manifest = conf.FindLatestManifest(train=train,
require_signature=True)
except Exception as e:
log.error("Could not find latest manifest due to %s" % str(e))
if new_manifest is None:
raise UpdateManifestNotFound("Manifest could not be found!")
# If new_manifest is not the requested train, then we don't have an update to do
if train and train != new_manifest.Train():
log.debug(
"CheckForUpdate(train = %s, cache_dir = %s): Wrong train in caache (%s)"
% (train, cache_dir, new_manifest.Train()))
return None
diffs = Manifest.DiffManifests(conf.SystemManifest(), new_manifest)
if diffs is None or len(diffs) == 0:
return None
log.debug("CheckForUpdate: diffs = %s" % diffs)
reboot = False
if handler and "Packages" in diffs:
for (pkg, op, old) in diffs["Packages"]:
handler(op, pkg, old)
# We attempt to see if the update requires a reboot.
# This is only tentative -- if we can't download a delta package,
# and that's what allows the update to avoid a reboot, a reboot
# is going to be required.
if op == "install":
if pkg.RequiresReboot() == True:
reboot = True
elif op == "upgrade":
upd = pkg.Update(old.Version())
if upd:
if upd.RequiresReboot():
reboot = True
elif pkg.RequiresReboot():
reboot = True
diffs["Reboot"] = reboot
if diff_handler:
diff_handler(diffs)
return new_manifest
def Update(root=None,
conf=None,
train=None,
check_handler=None,
get_handler=None,
install_handler=None,
cache_dir=None):
"""
Perform an update. If cache_dir is set, and contains a valid
set of files (MANIFEST exists, and is for the current train, and
the sequence is not the same as the system sequence), it will
use that, rather than downloading. It calls CheckForUpdates() otherwise,
to see if there are any. If there are, then magic happens.
"""
log.debug("Update(root = %s, conf = %s, train = %s, cache_dir = %s)" %
(root, conf, train, cache_dir))
if conf is None:
conf = Configuration.Configuration(root)
deleted_packages = []
process_packages = []
def UpdateHandler(op, pkg, old):
if op == "delete":
deleted_packages.append(pkg)
else:
process_packages.append(pkg)
if check_handler is not None:
check_handler(op, pkg, old)
new_man = None
if cache_dir:
if os.path.exists(cache_dir + "/MANIFEST"):
try:
cur = conf.SystemManifest()
new_man = Manifest.Manifest()
new_man.LoadPath(cache_dir + "/MANIFEST")
if new_man.Train() != cur.Train():
new_man = None
cache_dir = None
else:
diffs = Manifest.CompareManifests(cur, new_man)
for (pkg, op, old) in diffs:
UpdateHandler(op, pkg, old)
conf.SetPackageDir(cache_dir)
except Exception as e:
log.debug("Caught exception %s" % str(e))
new_man = None
cache_dir = None
if new_man is None:
new_man = CheckForUpdates(root, handler=UpdateHandler, train=train)
if new_man is None:
return
if len(deleted_packages) == 0 and len(process_packages) == 0:
# We have a case where a manifest was updated, but we
# don't actually have any changes to the packages. We
# should install the new manifest, and be done -- it
# may have new release notes, or other issues.
# Right now, I'm not quite sure how to do this.
log.debug("Updated manifest but no package differences")
return
# Now we have a list of deleted packages, and a list
# of update/install packages.
# The task is to delete the first set of packages,
# and then run through the others to install/update.
# First, however, we need to get the files.
# We want to use the system configuration, unless one has been
# specified -- we don't want to use the target root's.
if conf is None:
conf = Configuration.Configuration()
# If root is None, then we will try to create a clone
# environment. (If the caller wants to install into the
# current boot environment, set root = "" or "/".)
clone_name = None
mount_point = None
try:
if root is None:
# We clone the existing boot environment to
# "Avatar()-<sequence>"
clone_name = "%s-%s" % (Avatar(), new_man.Sequence())
if CreateClone(clone_name) is False:
log.error("Unable to create boot-environment %s" % clone_name)
raise Exception("Unable to create new boot-environment %s" %
clone_name)
mount_point = MountClone(clone_name)
if mount_point is None:
log.error("Unable to mount boot-environment %s" % clone_name)
DeleteClone(clone_name)
raise Exception("Unable to mount boot-environment %s" %
clone_name)
else:
root = mount_point
else:
mount_point = None
for pkg in deleted_packages:
log.debug("Want to delete package %s" % pkg.Name())
if conf.PackageDB(root).RemovePackageContents(pkg) == False:
log.error("Unable to remove contents package %s" % pkg.Name())
if mount_point:
UnmountClone(clone_name, mount_point)
mount_point = None
DeleteClone(clone_name)
raise Exception("Unable to remove contents for package %s" %
pkg.Name())
conf.PackageDB(root).RemovePackage(pkg.Name())
installer = Installer.Installer(manifest=new_man,
root=root,
config=conf)
installer.GetPackages(process_packages, handler=get_handler)
log.debug("Packages = %s" % installer._packages)
# Now let's actually install them.
# Only change on success
rv = False
if installer.InstallPackages(handler=install_handler) is False:
log.error("Unable to install packages")
else:
new_man.Save(root)
if mount_point is not None:
if UnmountClone(clone_name, mount_point) is False:
log.error("Unable to mount clone enivironment %s" %
clone_name)
else:
mount_point = None
if ActivateClone(clone_name) is False:
log.error("Could not activate clone environment %s" %
clone_name)
else:
# Downloaded package files are open-unlinked, so don't
# have to be cleaned up. If there's a cache directory,
# however, we need to get rid of it.
if cache_dir:
import shutil
if os.path.exists(cache_dir):
try:
shutil.rmtree(cache_dir)
except Exception as e:
# If that doesn't work, for now at least we'll
# simply ignore the error.
log.debug(
"Tried to remove cache directory %s, got exception %s"
% (cache_dir, str(e)))
rv = True
# Start a scrub. We ignore the return value.
RunCommand("/sbin/zpool", ["scrub", "freenas-boot"])
except BaseException as e:
log.error("Update got exception during update: %s" % str(e))
if clone_name:
if mount_point:
# Ignore the error here
UnmountClone(clone_name, mount_point)
DeleteClone(clone_name)
raise e
# Clean up
# That just leaves the clone, which
# we should unmount, and destroy if necessary.
# Unmounting attempts to delete the mount point that was created.
if rv is False:
if clone_name and DeleteClone(clone_name) is False:
log.error("Unable to delete boot environment %s in failure case" %
clone_name)
return rv
def main():
config_file = None
mani_file = None
remote_train = None
try:
opts, args = getopt.getopt(sys.argv[1:], "C:M:R:")
except getopt.GetoptError as err:
print str(err)
usage()
for o, a in opts:
if o == "-C":
config = a
elif o == "-M":
mani_file = a
elif o == "-R":
if "/" in a:
(remote_train, remote_manifest) = a.split("/")
if remote_manifest is not None and remote_manifest.startswith(
"FreeNAS-"):
remote_manifest = remote_manifest[len("FreeNAS-"):]
else:
usage()
if len(args) == 0:
usage()
conf = Configuration.Configuration(file=config_file)
if remote_train is not None:
mani = conf.GetManifest(remote_train, remote_manifest)
elif mani_file is not None:
mani = Manifest.Manifest(conf)
mani.LoadPath(mani_file)
else:
mani = conf.SystemManifest()
if mani is None:
print >> sys.stderr, "Unable to load manifest"
return (1)
if args[0] == "list":
list_cmd(mani, args[1:])
elif args[0] == "train":
print mani.Train()
elif args[0] == "sequence":
print mani.Sequence()
elif args[0] == "version":
if mani.Version() is not None:
print mani.Version()
elif args[0] == "notes":
if mani.Notes() is not None:
print mani.Notes()
elif args[0] == "show":
show_cmd(mani, args[1:])
elif args[0] == "sign":
sign_manifest(mani, args[1:])
else:
usage()
return 0
def ApplyUpdate(directory, install_handler=None, force_reboot=False):
"""
Apply the update in <directory>. As with PendingUpdates(), it will
have to verify the contents before it actually installs them, so
it has the same behaviour with incomplete or invalid content.
"""
rv = False
conf = Configuration.Configuration()
# Note that PendingUpdates may raise an exception
changes = PendingUpdatesChanges(directory)
if changes is None:
# This means no updates to apply, and so nothing to do.
return None
# Do I have to worry about a race condition here?
new_manifest = Manifest.Manifest(require_signature=True)
try:
new_manifest.LoadPath(directory + "/MANIFEST")
except ManifestInvalidSignature as e:
log.error("Cached manifest has invalid signature: %s" % str(e))
raise e
conf.SetPackageDir(directory)
# If we're here, then we have some change to make.
# PendingUpdatesChanges always sets this, unless it returns None
reboot = changes["Reboot"]
if force_reboot:
# Just in case
reboot = True
changes.pop("Reboot")
if len(changes) == 0:
# This shouldn't happen
log.debug("ApplyUupdate: changes only has Reboot key")
return None
deleted_packages = []
updated_packages = []
if "Packages" in changes:
for (pkg, op, old) in changes["Packages"]:
if op == "delete":
log.debug("Delete package %s" % pkg.Name())
deleted_packages.append(pkg)
continue
elif op == "install":
log.debug("Install package %s" % pkg.Name())
updated_packages.append(pkg)
elif op == "upgrade":
log.debug(
"Upgrade package %s-%s to %s-%s" %
(old.Name(), old.Version(), pkg.Name(), pkg.Version()))
updated_packages.append(pkg)
else:
log.error("Unknown package operation %s for %s" %
(op, pkg.Name()))
if new_manifest.Sequence().startswith(Avatar() + "-"):
new_boot_name = new_manifest.Sequence()
else:
new_boot_name = "%s-%s" % (Avatar(), new_manifest.Sequence())
log.debug("new_boot_name = %s, reboot = %s" % (new_boot_name, reboot))
mount_point = None
if reboot:
# Need to create a new boot environment
try:
if CreateClone(new_boot_name) is False:
log.debug("Failed to create BE %s" % new_boot_name)
# It's possible the boot environment already exists.
s = None
clones = ListClones()
if clones:
found = False
for c in clones:
if c["name"] == new_boot_name:
found = True
if c["mountpoint"] != "/":
if c["mountpoint"] != "-":
mount_point = c["mountpoint"]
else:
s = "Cannot create boot-environment with same name as current boot-environment (%s)" % new_boot_name
break
if found is False:
s = "Unable to create boot-environment %s" % new_boot_name
else:
log.debug("Unable to list clones after creation failure")
s = "Unable to create boot-environment %s" % new_boot_name
if s:
log.error(s)
raise UpdateBootEnvironmentException(s)
if mount_point is None:
mount_point = MountClone(new_boot_name)
except:
mount_point = None
s = sys.exc_info()[0]
if mount_point is None:
s = "Unable to mount boot-environment %s" % new_boot_name
log.error(s)
DeleteClone(new_boot_name)
raise UpdateBootEnvironmentException(s)
else:
# Need to do magic to move the current boot environment aside,
# and assign the newname to the current boot environment.
# Also need to make a snapshot of the current root so we can
# clean up on error
mount_point = None
log.debug("We should try to do a non-rebooty update")
root_dataset = GetRootDataset()
if root_dataset is None:
log.error("Unable to determine root environment name")
raise UpdateBootEnvironmentException(
"Unable to determine root environment name")
# We also want the root name
root_env = None
clones = ListClones()
if clones is None:
log.error("Unable to determine root BE")
raise UpdateBootEnvironmentException("Unable to determine root BE")
for clone in clones:
if clone["mountpoint"] == "/":
root_env = clone
break
if root_env is None:
log.error("Unable to find root BE!")
raise UpdateBootEnvironmentException("Unable to find root BE!")
# Now we want to snapshot the current boot environment,
# so we can rollback as needed.
snapshot_name = "%s@Pre-Uprgade-%s" % (root_dataset,
new_manifest.Sequence())
cmd = "/sbin/zfs"
args = ["snapshot", "-r", snapshot_name]
rv = RunCommand(cmd, args)
if rv is False:
log.error("Unable to create snapshot %s, bailing for now" %
snapshot_name)
raise UpdateSnapshotException("Unable to create snapshot %s" %
snapshot_name)
# We need to remove the beadm:nickname property. I hate knowing this much
# about the implementation
args = ["inherit", "-r", "beadm:nickname", snapshot_name]
RunCommand(cmd, args)
# At this point, we'd want to rename the boot environment to be the new
# name, which would be new_manifest.Sequence()
if CreateClone(new_boot_name, rename=root_env["name"]) is False:
log.error("Unable to create new boot environment %s" %
new_boot_name)
# Roll back and destroy the snapshot we took
cmd = "/sbin/zfs"
args = ["rollback", snapshot_name]
RunCommand(cmd, args)
args[0] = "destroy"
RunCommand(cmd, args)
# And set the beadm:nickname property back
args = ["set", "beadm:nickname=%s" % root_env["name"]]
RunCommand(cmd, args)
raise UpdateBootEnvironmentException(
"Unable to create new boot environment %s" % new_boot_nam)
# Now we start doing the update!
# If we have to reboot, then we need to
# make a new boot environment, with the appropriate name.
# If we are *not* rebooting, then we want to rename the
# current one with the appropriate name, while at the same
# time cloning the current one and keeping the existing name.
# Easy peasy, right?
try:
# Remove any deleted packages
for pkg in deleted_packages:
log.debug("About to delete package %s" % pkg.Name())
if conf.PackageDB(mount_point).RemovePackageContents(pkg) == False:
s = "Unable to remove contents for packate %s" % pkg.Name()
if mount_point:
UnmountClone(new_boot_name, mount_point)
mount_point = None
DeleteClone(new_boot_name)
raise UpdatePackageException(s)
conf.PackageDB(mount_point).RemovePackage(pkg.Name())
installer = Installer.Installer(manifest=new_manifest,
root=mount_point,
config=conf)
installer.GetPackages(pkgList=updated_packages)
log.debug("Installer got packages %s" % installer._packages)
# Now to start installing them
rv = False
if installer.InstallPackages(handler=install_handler) is False:
log.error("Unable to install packages")
raise UpdatePackageException("Unable to install packages")
else:
new_manifest.Save(mount_point)
if mount_point:
if UnmountClone(new_boot_name, mount_point) is False:
s = "Unable to unmount clone environment %s from mount point %s" % (
new_boot_name, mount_point)
log.error(s)
raise UpdateBootEnvironmentException(s)
mount_point = None
if reboot:
if ActivateClone(new_boot_name) is False:
s = "Unable to activate clone environment %s" % new_boot_name
log.error(s)
raise UpdateBootEnvironmentException(s)
if not reboot:
# Clean up the emergency holographic snapshot
cmd = "/sbin/zfs"
args = ["destroy", "-r", snapshot_name]
rv = RunCommand(cmd, args)
if rv is False:
log.error("Unable to destroy snapshot %s" % snapshot_name)
RemoveUpdate(directory)
# RunCommand("/sbin/zpool", ["scrub", "freenas-boot"])
except BaseException as e:
# Cleanup code is entirely different for reboot vs non reboot
log.error("Update got exception during update: %s" % str(e))
if reboot:
if mount_point:
UnmountClone(new_boot_name, mount_point)
if new_boot_name:
DeleteClone(new_boot_name)
else:
# Need to roll back
# We also need to delete the renamed clone of /,
# and then rename / to the original name.
# First, however, destroy the clone
rv = DeleteClone(root_env["name"])
if rv:
# Next, rename the clone
rv = RenameClone(new_boot_name, root_env["name"])
if rv:
# Now roll back the snapshot, and set the beadm:nickname value
cmd = "/sbin/zfs"
args = ["rollback", "-r", snapshot_name]
rv = RunCommand(cmd, args)
if rv is False:
log.error("Unable to rollback %s" % snapshot_name)
# Don't know what to do then
args = [
"set",
"beadm:nickname=%s" % root_env["name"],
"freenas-boot/ROOT/%s" % root_env["name"]
]
rv = RunCommand(cmd, args)
if rv is False:
log.error(
"Unable to set nickname, wonder what I did wrong")
args = ["destroy", "-r", snapshot_name]
rv = RunCommand(cmd, args)
if rv is False:
log.error("Unable to destroy snapshot %s" %
snapshot_name)
raise e
return reboot
async def update(self, job, attrs=None):
"""
Downloads (if not already in cache) and apply an update.
"""
attrs = attrs or {}
trains = await self.middleware.call('update.get_trains')
train = attrs.get('train') or trains['selected']
try:
result = compare_trains(trains['current'], train)
except Exception:
self.logger.warning("Failed to compare trains %r and %r",
trains['current'],
train,
exc_info=True)
else:
errors = {
CompareTrainsResult.NIGHTLY_DOWNGRADE:
textwrap.dedent("""\
You're not allowed to change away from the nightly train, it is considered a downgrade.
If you have an existing boot environment that uses that train, boot into it in order to upgrade
that train.
"""),
CompareTrainsResult.MINOR_DOWNGRADE:
textwrap.dedent("""\
Changing minor version is considered a downgrade, thus not a supported operation.
If you have an existing boot environment that uses that train, boot into it in order to upgrade
that train.
"""),
CompareTrainsResult.MAJOR_DOWNGRADE:
textwrap.dedent("""\
Changing major version is considered a downgrade, thus not a supported operation.
If you have an existing boot environment that uses that train, boot into it in order to upgrade
that train.
"""),
}
if result in errors:
raise CallError(errors[result])
location = await self.middleware.call('update.get_update_location')
job.set_progress(0, 'Retrieving update manifest')
handler = UpdateHandler(self, job)
update = Update.DownloadUpdate(
train,
location,
check_handler=handler.check_handler,
get_handler=handler.get_handler,
)
if update is False:
raise ValueError('No update available')
new_manifest = Manifest.Manifest(require_signature=True)
new_manifest.LoadPath('{}/MANIFEST'.format(location))
Update.ApplyUpdate(
location,
install_handler=handler.install_handler,
)
await self.middleware.call('cache.put', 'update.applied', True)
if attrs.get('reboot'):
await self.middleware.call('system.reboot', {'delay': 10})
return True
def VerifyUpdate(directory):
"""
Verify the update in the directory is valid -- the manifest
is sane, any signature is valid, the package files necessary to
update are present, and have a valid checksum. Returns either
a file object if it's valid (the file object is locked), None
if it doesn't exist, or it raises an exception -- one of
UpdateIncompleteCacheException or UpdateInvalidCacheException --
if necessary.
"""
import fcntl
# First thing we do is get the systen configuration and
# systen manifest
conf = Configuration.Configuration()
mani = conf.SystemManifest()
# Next, let's see if the directory exists.
if not os.path.exists(directory):
return None
# Open up the manifest file. Assuming it exists.
try:
mani_file = open(directory + "/MANIFEST", "r+")
except:
# Doesn't exist. Or we can't get to it, which would be weird.
return None
# Let's try getting an exclusive lock on the manifest
try:
fcntl.lockf(mani_file, fcntl.LOCK_EX | fcntl.LOCK_NB, 0, 0)
except:
# Well, if we can't acquire the lock, someone else has it.
# Throw an incomplete exception
raise UpdateBusyCacheException("Cache directory %s is being modified" %
directory)
cached_mani = Manifest.Manifest()
try:
cached_mani.LoadFile(mani_file)
except Exception as e:
# If we got an exception, it's invalid.
log.error("Could not load cached manifest file: %s" % str(e))
raise UpdateInvalidCacheException
# First easy thing to do: look for the SEQUENCE file.
try:
cached_sequence = open(directory + "/SEQUENCE", "r").read().rstrip()
except (IOError, Exception) as e:
log.error("Could not sequence file in cache directory %s: %s" %
(directory, str(e)))
raise UpdateIncompleteCacheException(
"Cache directory %s does not have a sequence file" % directory)
# Now let's see if the sequence matches us.
if cached_sequence != mani.Sequence():
log.error("Cached sequence, %s, does not match system sequence, %s" %
(cached_sequence, mani.Sequence()))
raise UpdateInvalidCacheException(
"Cached sequence does not match system sequence")
# Next thing to do is go through the manifest, and decide which package files we need.
diffs = Manifest.CompareManifests(mani, cached_mani)
# This gives us an array to examine.
for (pkg, op, old) in diffs:
if op == "delete":
# Deleted package, so we don't need to do any verification here
continue
if op == "install":
# New package, being installed, so we need the full package
cur_vers = None
if op == "upgrade":
# Package being updated, so we can look for the delta package.
cur_vers = old.Version()
new_vers = pkg.Version()
# This is slightly redundant -- if cur_vers is None, it'll check
# the same filename twice.
if not os.path.exists(directory + "/" + pkg.FileName()) and \
not os.path.exists(directory + "/" + pkg.FileName(cur_vers)):
# Neither exists, so incoplete
log.error("Cache %s directory missing files for package %s" %
(directory, pkg.Name()))
raise UpdateIncompleteCacheException(
"Cache directory %s missing files for package %s" %
(directory, pkg.Name()))
# Okay, at least one of them exists.
# Let's try the full file first
try:
with open(directory + "/" + pkg.FileName()) as f:
if pkg.Checksum():
cksum = Configuration.ChecksumFile(f)
if cksum == pkg.Checksum():
continue
else:
continue
except:
pass
if cur_vers is None:
e = "Cache directory %s missing files for package %s" % (
directory, pkg.Name())
log.error(e)
raise UpdateIncompleteCacheException(e)
# Now we try the delta file
# To do that, we need to find the right dictionary in the pkg
upd_cksum = None
found = False
for update_dict in pkg.Updates():
if update_dict[Package.VERSION_KEY] == cur_vers:
if Package.CHECKSUM_KEY in update_dict:
upd_cksum = update_dict[Package.CHECKSUM_KEY]
try:
with open(directory + "/" +
pkg.FileName(cur_vers)) as f:
cksum = Configuration.ChecksumFile(f)
if upd_cksum == cksum:
found = True
break
except:
pass
else:
found = True
break
if found is False:
# If we got here, we are missing this file
log_msg = "Cache directory %s is missing package %s" % (directory,
pkg.Name())
log.error(log_msg)
raise UpdateIncompleteCacheException(log_msg)
# And if we got here, then we have found all of the packages, the manifest is fine,
# and the sequence tag is correct.
mani_file.seek(0)
return mani_file
if len(args) != 1:
usage()
root = args[0]
config = Configuration.SystemConfiguration()
if package_dir is not None:
config.SetPackageDir(package_dir)
if mani_file is None:
manifest = config.SystemManifest()
else:
# We ignore the signature because freenas-install is
# called from the ISO install, and the GUI install, which
# have their own checksums elsewhere.
manifest = Manifest.Manifest(config, require_signature=False)
manifest.LoadPath(mani_file)
installer = Installer.Installer(manifest=manifest,
root=root,
config=config)
if installer.GetPackages() is not True:
print("Huh, could not install and yet it returned", file=sys.stderr)
with Installer.ProgressHandler() as pf:
# For installation, we assume that we're running the same kernel as the new system.
installer.trampoline = False
installer.InstallPackages(progressFunc=pf.update,
handler=install_handler)