def send_telemetry(self):
logger.info(
'Uploading telemetry data to {0}'.format(TELEMETRY_ENDPOINT_PATH))
conf = Configuration.Configuration()
conf.LoadTrainsConfig()
manifest = conf.SystemManifest()
headers = {
'X-iXSystems-Project': Configuration.Avatar(),
'X-iXSystems-Version': manifest.Version(),
'X-iXSystems-HostID': self.hostuuid,
'X-iXSystems-Train': conf.CurrentTrain()
}
manifest = {
'host_uuid': self.hostuuid,
'cpu_type': sysctl.sysctlbyname("hw.model"),
'cpu_clock': sysctl.sysctlbyname("hw.clockrate"),
'cpu_cores': sysctl.sysctlbyname("hw.ncpu"),
'hypervisor': sysctl.sysctlbyname("kern.vm_guest"),
'mem_size': sysctl.sysctlbyname("hw.physmem")
}
files = {
name: open(os.path.join(TELEMETRY_STAGING_PATH, name), 'rb')
for name in os.listdir(TELEMETRY_STAGING_PATH)
}
files['manifest'] = (None, json.dumps(manifest), 'application/json')
try:
requests.post(TELEMETRY_ENDPOINT_PATH,
headers=headers,
files=files)
except BaseException as err:
logger.error('Cannot send telemerty data: {0}'.format(str(err)))
def CheckForUpdates(root = None):
"""
Check for an updated manifest.
Very simple, uses the configuration module.
Returns the new manifest if there is an update,
and None otherwise.
"""
conf = Configuration.Configuration()
cur = Configuration.SystemManifest(root)
m = conf.FindNewerManifest(cur.Sequence())
if verbose > 1 or debug > 0:
print >> sys.stderr, "Current sequence = %d, available sequence = %d" % (cur.Sequence(), m.Sequence() if m is not None else 0)
return m
def get_trains_data(self):
try:
redir_trains = self._get_redir_trains()
except Exception:
self.logger.warn('Failed to retrieve trains redirection',
exc_info=True)
redir_trains = {}
conf = Configuration.Configuration()
conf.LoadTrainsConfig()
trains = {}
for name, descr in (conf.AvailableTrains() or {}).items():
train = conf._trains.get(name)
if train is None:
train = Train.Train(name, descr)
trains[train.Name()] = {
'description': descr,
'sequence': train.LastSequence(),
}
return {
'trains': trains,
'current_train': conf.CurrentTrain(),
'trains_redirection': redir_trains,
}
def check_train(self, train):
handler = CheckUpdateHandler()
manifest = CheckForUpdates(
diff_handler=handler.diff_call,
handler=handler.call,
train=train,
)
if not manifest:
return {'status': 'UNAVAILABLE'}
data = {
'status': 'AVAILABLE',
'changes': handler.changes,
'notice': manifest.Notice(),
'notes': manifest.Notes(),
}
conf = Configuration.Configuration()
sys_mani = conf.SystemManifest()
if sys_mani:
sequence = sys_mani.Sequence()
else:
sequence = ''
data['changelog'] = get_changelog(train,
start=sequence,
end=manifest.Sequence())
data['version'] = manifest.Version()
return data
def version(self):
if self.__version is None:
conf = Configuration.Configuration()
sys_mani = conf.SystemManifest()
if sys_mani:
self.__version = sys_mani.Version()
return self.__version
def get_trains(self):
"""
Returns available trains dict and the currently configured train as well as the
train of currently booted environment.
"""
data = self.middleware.call_sync('datastore.config', 'system.update')
conf = Configuration.Configuration()
conf.LoadTrainsConfig()
selected = None
trains = {}
for name, descr in (conf.AvailableTrains() or {}).items():
train = conf._trains.get(name)
if train is None:
train = Train.Train(name, descr)
if not selected and data['upd_train'] == train.Name():
selected = data['upd_train']
trains[train.Name()] = {
'description': train.Description(),
'sequence': train.LastSequence(),
}
if not data['upd_train'] or not selected:
selected = conf.CurrentTrain()
return {
'trains': trains,
'current': conf.CurrentTrain(),
'selected': selected,
}
def PendingUpdates(directory):
"""
Return a list (a la CheckForUpdates handler right now) of
changes between the currently installed system and the
downloaded contents in <directory>. If <directory>'s values
are incomplete or invalid for whatever reason, return
None. "Incomplete" means a necessary file for upgrading
from the current system is not present; "Invalid" means that
one part of it is invalid -- manifest is not valid, signature isn't
valid, checksum for a file is invalid, or the stashed sequence
number does not match the current system's sequence.
"""
mani_file = None
conf = Configuration.Configuration()
try:
mani_file = VerifyUpdate(directory)
except UpdateBusyCacheException:
log.debug("Cache directory %s is busy, so no update available" %
directory)
return None
except (UpdateIncompleteCacheException, UpdateInvalidCacheException) as e:
log.error(str(e))
RemoveUpdate(directory)
return None
except BaseException as e:
log.error(
"Got exception %s while trying to determine pending updates" %
str(e))
return None
if mani_file:
new_manifest = Manifest.Manifest()
new_manifest.LoadFile(mani_file)
diffs = Manifest.CompareManifests(conf.SystemManifest(), new_manifest)
return diffs
return None
def check_available(self, attrs=None):
"""
Checks if there is an update available from update server.
status:
- REBOOT_REQUIRED: an update has already been applied
- AVAILABLE: an update is available
- UNAVAILABLE: no update available
.. examples(websocket)::
Check available update using default train:
:::javascript
{
"id": "6841f242-840a-11e6-a437-00e04d680384",
"msg": "method",
"method": "update.check_available"
}
"""
try:
applied = self.middleware.call_sync('cache.get', 'update.applied')
except Exception:
applied = False
if applied is True:
return {'status': 'REBOOT_REQUIRED'}
train = (attrs or {}).get('train') or self.middleware.call_sync(
'update.get_trains')['selected']
handler = CheckUpdateHandler()
manifest = CheckForUpdates(
diff_handler=handler.diff_call,
handler=handler.call,
train=train,
)
if not manifest:
return {'status': 'UNAVAILABLE'}
data = {
'status': 'AVAILABLE',
'changes': handler.changes,
'notice': manifest.Notice(),
'notes': manifest.Notes(),
}
conf = Configuration.Configuration()
sys_mani = conf.SystemManifest()
if sys_mani:
sequence = sys_mani.Sequence()
else:
sequence = ''
data['changelog'] = get_changelog(train,
start=sequence,
end=manifest.Sequence())
data['version'] = manifest.Version()
return data
def download(self, job):
train = self.middleware.call_sync('update.get_trains')['selected']
location = self.middleware.call_sync('update.get_update_location')
job.set_progress(0, 'Retrieving update manifest')
handler = UpdateHandler(self, job, 100)
Update.DownloadUpdate(
train,
location,
check_handler=handler.check_handler,
get_handler=handler.get_handler,
)
update = Update.CheckForUpdates(train=train, cache_dir=location)
if not update:
return False
notified = False
try:
if self.middleware.call_sync('cache.has_key', 'update.notified'):
notified = self.middleware.call_sync('cache.get',
'update.notified')
except Exception:
pass
if not notified:
self.middleware.call_sync('cache.put', 'update.notified', True)
conf = Configuration.Configuration()
sys_mani = conf.SystemManifest()
if sys_mani:
sequence = sys_mani.Sequence()
else:
sequence = ''
changelog = get_changelog(train,
start=sequence,
end=update.Sequence())
try:
# FIXME: Translation
self.middleware.call_sync(
'mail.send', {
'subject': 'Update Available',
'text':
'''A new update is available for the %(train)s train.
Version: %(version)s
Changelog:
%(changelog)s
''' % {
'train': train,
'version': update.Version(),
'changelog': changelog,
},
}).wait_sync()
except Exception:
self.logger.warn('Failed to send email about new update',
exc_info=True)
return True
def CheckForUpdates(root = None, handler = None):
"""
Check for an updated manifest.
Very simple, uses the configuration module.
Returns the new manifest if there is an update,
and None otherwise.
(It determines if there is an update if the latest-found
manifest contains differences from the current system
manifest.)
The optional argument handler is a function that
will be called for each difference in the new manifest
(if there is one); it will be called with three
arguments: operation, package, old package.
operation will be "delete", "upgrade", or "install";
old package will be None for delete and install.
"""
conf = Configuration.Configuration(root)
cur = conf.SystemManifest()
m = conf.FindLatestManifest()
log.debug("Current sequence = %s, available sequence = %s" % (cur.Sequence(), m.Sequence()
if m is not None else "None"))
if m is None:
raise ValueError("Manifest could not be found!")
diffs = Manifest.CompareManifests(cur, m)
update = False
for (pkg, op,old) in diffs:
update = True
if handler is not None:
handler(op, pkg, old)
return m if update else None
def trains(self):
conf = Configuration.Configuration()
conf.LoadTrainsConfig()
trains = conf.AvailableTrains()
if trains is None:
logger.debug(
'The AvailableTrains call returned None. Check your network connection'
)
return None
seltrain = self.dispatcher.configstore.get('update.train')
data = []
for name in list(trains.keys()):
if name in conf._trains:
train = conf._trains.get(name)
else:
train = Train.Train(name)
data.append({
'name': train.Name(),
'description': train.Description(),
'sequence': train.LastSequence(),
'current': True if name == seltrain else False,
})
return data
def get_changelog(train, start='', end=''):
conf = Configuration.Configuration()
changelog = conf.GetChangeLog(train=train)
if not changelog:
return None
return parse_changelog(changelog.read(), start, end)
def sw_version_is_stable():
conf = Configuration.Configuration()
train = conf.CurrentTrain()
if train and 'stable' in train.lower():
return True
else:
return False
def CheckForUpdates(root = None, handler = None):
"""
Check for an updated manifest.
Very simple, uses the configuration module.
Returns the new manifest if there is an update,
and None otherwise.
(It determines if there is an update if the latest-found
manifeset's sequence number is larger than the current
sequence number.)
The optional argument handler is a function that
will be called for each difference in the new manifest
(if there is one); it will be called with three
arguments: operation, package, old package.
operation will be "delete", "upgrade", or "install";
old package will be None for delete and install.
"""
conf = Configuration.Configuration(root)
cur = conf.SystemManifest()
m = conf.FindLatestManifest()
print >> sys.stderr, "Current sequence = %d, available sequence = %d" % (cur.Sequence(), m.Sequence() if m is not None else 0)
if m is not None and m.Sequence() > cur.Sequence():
if handler is not None:
diffs = Manifest.CompareManifests(cur, m)
for (pkg, op, old) in diffs:
handler(op, pkg, old)
return m
return None
def _system_info(request=None):
# OS, hostname, release
__, hostname, __ = os.uname()[0:3]
platform = sysctl.filter('hw.model')[0].value
physmem = '%dMB' % (sysctl.filter('hw.physmem')[0].value / 1048576, )
# All this for a timezone, because time.asctime() doesn't add it in.
date = time.strftime('%a %b %d %H:%M:%S %Z %Y') + '\n'
uptime = subprocess.check_output(
"env -u TZ uptime | awk -F', load averages:' '{ print $1 }'",
shell=True)
loadavg = "%.2f, %.2f, %.2f" % os.getloadavg()
try:
freenas_build = '%s %s' % (get_sw_name(), get_sw_login_version())
except:
freenas_build = "Unrecognized build"
try:
conf = Configuration.Configuration()
manifest = conf.SystemManifest()
builddate = datetime.utcfromtimestamp(int(manifest.Sequence()))
except:
builddate = None
return {
'hostname': hostname,
'platform': platform,
'physmem': physmem,
'date': date,
'uptime': uptime,
'loadavg': loadavg,
'freenas_build': freenas_build,
'builddate': builddate,
}
def Install(root = None, manifest = None):
"""
Perform an install. Uses the system manifest, and installs
into root. root must be set.
"""
if root is None:
print >> sys.stderr, "Install must have target root specified"
usage()
conf = Configuration.Configuration()
if manifest is not None:
cur = Manifest.Manifest()
try:
cur.LoadPath(manifest)
except Exception as e:
print >> sys.stderr, "Could not load manifest from %s: %s" % (manifest, str(e))
return False
else:
try:
cur = Configuration.SystemManifest()
except:
print >> sys.stderr, "Cannot get system manifest"
return False
if cur is None or cur.Packages() is None:
raise Exception("Cannot load configuration")
print "Want to install into %s" % root
#
# To install, we have to grab each package in the manifest,
# and install them into the specified root directory.
# When we are done, we write out the system manifest into
# the manifest directory.
for pkg in cur.Packages():
print "Package %s" % pkg.Name()
filename = Manifest.FormatName(pkg.Name(), pkg.Version())
f = conf.FindPackage(filename, pkg.Checksum())
if f is None:
print >> sys.stderr, "\tCould not find package file for %s" % filename
return False
else:
if Installer.install_file(f, root) == False:
print >> sys.stderr, "Could not install package %s" % filename
return False
else:
print "%s installed" % pkg.Name()
conf.StoreManifest(cur, root)
return True
def sw_version_is_stable():
conf = Configuration.Configuration()
if 'stable' in conf.CurrentTrain().lower():
return True
else:
return False
def sw_buildtime():
global BUILDTIME
if BUILDTIME is None:
conf = Configuration.Configuration()
sys_mani = conf.SystemManifest()
if sys_mani:
BUILDTIME = sys_mani.TimeStamp()
return BUILDTIME
def sw_version():
global VERSION
if VERSION is None:
conf = Configuration.Configuration()
sys_mani = conf.SystemManifest()
if sys_mani:
VERSION = sys_mani.Version()
return VERSION
def get_config(self):
configuration = Configuration.Configuration()
return {
'train': self.dispatcher.configstore.get('update.train'),
'check_auto': self.dispatcher.configstore.get('update.check_auto'),
'internal': configuration.UpdateServerName() == 'internal',
'update_server': configuration.UpdateServerURL(),
}
def get_changelog(train, cache_dir='/var/tmp/update', start='', end=''):
"Utility to get and eventually parse a changelog if available"
conf = Configuration.Configuration()
changelog = conf.GetChangeLog(train=train, save_dir=cache_dir)
if not changelog:
return None
return parse_changelog(changelog.read(), start, end)
def main():
try:
updateobj = mUpdate.objects.order_by('-id')[0]
except IndexError:
updateobj = mUpdate.objects.create()
if updateobj.upd_autocheck is False:
return
location = notifier().get_update_location()
Update.DownloadUpdate(updateobj.get_train(), location)
update = Update.CheckForUpdates(
train=updateobj.get_train(),
cache_dir=location,
)
if not update:
return
conf = Configuration.Configuration()
sys_mani = conf.SystemManifest()
if sys_mani:
sequence = sys_mani.Sequence()
else:
sequence = ''
changelog = get_changelog(
updateobj.get_train(),
start=sequence,
end=update.Sequence(),
)
hostname = socket.gethostname()
send_mail(
subject='%s: %s' % (
hostname,
_('Update Available'),
),
extra_headers={
'X-Mailer': get_sw_name(),
'X-%s-Host' % get_sw_name(): socket.gethostname()
},
text=_('''A new update is available for the %(train)s train.
Version: %(version)s
Changelog:
%(changelog)s
''') % {
'train': updateobj.get_train(),
'version': update.Version(),
'changelog': changelog,
},
)
def sw_version_is_stable():
# Lazy import to avoid freenasOS configure logging for us
from freenasOS import Configuration
conf = Configuration.Configuration()
train = conf.CurrentTrain()
if train and 'stable' in train.lower():
return True
else:
return False
async def download(self, job):
train = (await self.get_trains())['selected']
location = await self.middleware.call('notifier.get_update_location')
Update.DownloadUpdate(
train,
location,
)
update = Update.CheckForUpdates(train=train, cache_dir=location)
if not update:
return False
notified = False
try:
if await self.middleware.call('cache.has_key', 'update.notified'):
notified = await self.middleware.call('cache.get',
'update.notified')
except Exception:
pass
if not notified:
await self.middleware.call('cache.put', 'update.notified', True)
conf = Configuration.Configuration()
sys_mani = conf.SystemManifest()
if sys_mani:
sequence = sys_mani.Sequence()
else:
sequence = ''
changelog = get_changelog(train,
start=sequence,
end=update.Sequence())
hostname = socket.gethostname()
try:
# FIXME: Translation
await self.middleware.call(
'mail.send', {
'subject': '{}: {}'.format(hostname,
'Update Available'),
'text':
'''A new update is available for the %(train)s train.
Version: %(version)s
Changelog:
%(changelog)s
''' % {
'train': train,
'version': update.Version(),
'changelog': changelog,
},
})
except Exception:
self.logger.warn('Failed to send email about new update',
exc_info=True)
return True
def sw_buildtime():
# Lazy import to avoid freenasOS configure logging for us
from freenasOS import Configuration
global BUILDTIME
if BUILDTIME is None:
conf = Configuration.Configuration()
sys_mani = conf.SystemManifest()
if sys_mani:
BUILDTIME = sys_mani.TimeStamp()
return BUILDTIME
def sw_version():
# Lazy import to avoid freenasOS configure logging for us
from freenasOS import Configuration
global VERSION
if VERSION is None:
conf = Configuration.Configuration()
sys_mani = conf.SystemManifest()
if sys_mani:
VERSION = sys_mani.Version()
return VERSION
def run(self, path, reboot_post_install=False):
self.set_progress(0, 'Extracting update from tarfile...')
cache_dir = self.dispatcher.call_sync('update.update_cache_getter',
'cache_dir')
if cache_dir is None:
try:
cache_dir = self.dispatcher.call_sync(
'system_dataset.request_directory', 'update')
except RpcException:
cache_dir = '/var/tmp/update'
# Frozen tarball. We'll extract it into the cache directory, and
# then add a couple of things to make it pass sanity, and then apply it.
# For now we just copy the code above.
# First, remove the cache directory
# Hrm, could overstep a locked file.
shutil.rmtree(cache_dir, ignore_errors=True)
try:
os.makedirs(cache_dir)
except BaseException as e:
raise TaskException(
errno.EPERM,
"Unable to create cache directory {0}: {1}".format(
cache_dir, str(e)))
try:
with tarfile.open(path) as tf:
files = tf.getmembers()
for f in files:
if f.name in ("./", ".", "./."):
continue
if not f.name.startswith("./"):
continue
if len(f.name.split("/")) != 2:
continue
tf.extract(f.name, path=cache_dir)
except BaseException as e:
raise TaskException(
errno.EIO, "Unable to extract frozen update {0}: {1}".format(
path, str(e)))
config = Configuration.SystemConfiguration()
# Exciting! Now we need to have a SEQUENCE file, or it will fail verification.
with open(os.path.join(cache_dir, "SEQUENCE"), "w") as s:
s.write(config.SystemManifest().Sequence())
# And now the SERVER file
with open(os.path.join(cache_dir, "SERVER"), "w") as s:
s.write(config.UpdateServerName())
# Now we can go for the update apply task
self.run_subtask_sync('update.apply',
reboot_post_install,
progress_callback=lambda p, m=
'Installing updates from extracted tarfile', e=
None: self.chunk_progress(0, 100, '', p, m, e))
def get_train(self):
# FIXME: lazy import, why?
from freenasOS import Configuration
conf = Configuration.Configuration()
conf.LoadTrainsConfig()
trains = conf.AvailableTrains() or []
if trains:
trains = list(trains.keys())
if not self.upd_train or self.upd_train not in trains:
return conf.CurrentTrain()
return self.upd_train
def get_train(self):
"""
Returns currently configured train
"""
data = self.middleware.call('datastore.config', 'system.update')
conf = Configuration.Configuration()
conf.LoadTrainsConfig()
trains = conf.AvailableTrains() or []
if trains:
trains = trains.keys()
if not data['upd_train'] or data['upd_train'] not in trains:
return conf.CurrentTrain()
def get_trains(self):
"""
Returns available trains dict and the currently configured train as well as the
train of currently booted environment.
"""
data = self.middleware.call_sync('datastore.config', 'system.update')
conf = Configuration.Configuration()
conf.LoadTrainsConfig()
try:
redir_trains = self._get_redir_trains()
except Exception:
self.logger.warn('Failed to retrieve trains redirection',
exc_info=True)
redir_trains = {}
selected = None
trains = {}
for name, descr in (conf.AvailableTrains() or {}).items():
train = conf._trains.get(name)
if train is None:
train = Train.Train(name, descr)
try:
result = compare_trains(conf.CurrentTrain(), train.Name())
except Exception:
self.logger.warning("Failed to compare trains %r and %r",
conf.CurrentTrain(),
train.Name(),
exc_info=True)
continue
else:
if result in BAD_UPGRADES:
continue
if not selected and data['upd_train'] == train.Name():
selected = data['upd_train']
if name in redir_trains:
continue
trains[train.Name()] = {
'description': descr,
'sequence': train.LastSequence(),
}
if not data['upd_train'] or not selected:
selected = conf.CurrentTrain()
if selected in redir_trains:
selected = redir_trains[selected]
return {
'trains': trains,
'current': conf.CurrentTrain(),
'selected': selected,
}
def run(self):
try:
error_flag, ed, warn_flag, wl = Configuration.do_verify(self.verify_handler)
except Exception as e:
raise TaskException(
errno.EAGAIN, 'Got exception while verifying install: {0}'.format(str(e))
)
return {
'checksum': ed['checksum'],
'notfound': ed['notfound'],
'wrongtype': ed['wrongtype'],
'perm': wl,
'error': error_flag,
'warn': warn_flag,
}
def update_verify(request):
if request.method == 'POST':
handler = VerifyHandler()
try:
log.debug("Starting VerifyUpdate")
error_flag, ed, warn_flag, wl = Configuration.do_verify(handler.verify_handler)
except Exception, e:
log.debug("VerifyUpdate Exception ApplyUpdate: %s" %e)
handler.error = unicode(e)
handler.finished = True
handler.dump()
log.debug("VerifyUpdate finished!")
if handler.error is not False:
handler.exit()
raise MiddlewareError(handler.error)
handler.exit()
if error_flag or warn_flag:
checksums = None
wrongtype = None
notfound = None
perms = None
if ed['checksum']:
checksums = ed['checksum']
if ed['notfound']:
notfound = ed['notfound']
if ed['wrongtype']:
wrongtype = ed['wrongtype']
if warn_flag:
perms = wl
return render(request, 'system/update_verify.html', {
'error': True,
'checksums': checksums,
'notfound': notfound,
'wrongtype': wrongtype,
'perms': perms,
})
else:
return render(request, 'system/update_verify.html', {
'success': True,
})
#!/usr/local/bin/python3
import sys
import traceback
sys.path.append("/usr/local/lib")
from freenasOS import Configuration
if __name__ == '__main__':
try:
error_flag, ed, warn_flag, wl = Configuration.do_verify()
except IOError as e:
traceback.print_exc()
sys.exit(74)
if error_flag or warn_flag:
print("The following inconsistencies were found in your current install:")
if ed['checksum']:
print("\nList of Checksum Mismatches:\n")
for entry in ed['checksum']:
print(entry["path"])
if ed['notfound']:
print("\nList of Files/Directories/Symlinks not Found:\n")
for entry in ed['notfound']:
print(entry["path"])
if ed['wrongtype']:
print("\nList of Incorrect Filetypes:\n")
for entry in ed['wrongtype']:
print(entry["path"], "\t" , entry["problem"])