def check(self, fast=False, progress=lambda x: ()):
""" Check a package against it's known checksum, or if fast is
specified, just check that the package exists. """
if fast:
return self.repository.accessor().access(self.name)
else:
try:
logger.log("Validating package %s" % self.name)
namefp = self.repository.accessor().openAddress(self.name)
m = hashlib.sha256()
data = ''
total_read = 0
while True:
data = namefp.read(10485760)
total_read += len(data)
if data == '':
break
else:
m.update(data)
progress(total_read / (self.size / 100))
namefp.close()
calculated = m.hexdigest()
valid = (self.sha256sum == calculated)
return valid
except Exception as e:
return False
def bindMount(source, mountpoint):
logger.log("Bind mounting %s to %s" % (source, mountpoint))
cmd = [ '/bin/mount', '--bind', source, mountpoint]
rc, out, err = runCmd2(cmd, with_stdout=True, with_stderr=True)
if rc != 0:
raise MountFailureException("out: '%s' err: '%s'" % (out, err))
def doInteractiveLoadDriver(ui, answers):
media = None
address = None
required_repo_list = []
loaded_drivers = []
rc = ui.init.driver_disk_sequence(answers, answers['driver-repos'])
if rc:
media, address = rc
repos = answers['repos']
# now load the drivers:
for r in repos:
logger.log("Processing repo %s" % r)
try:
r.installPackages(lambda x: (), {'root': '/'})
answers['driver-repos'].append(str(r))
ButtonChoiceWindow(ui.screen, "Drivers Loaded",
"Loaded %s." % r.name(), ['Ok'])
except Exception as e:
logger.logException(e)
ButtonChoiceWindow(
ui.screen, "Problem Loading Driver",
"Setup was unable to load the device driver.", ['Ok'])
return media, address
def run_script(script, stage, *args):
logger.log("Running script for stage %s: %s %s" % (stage, script, ' '.join(args)))
util.assertDir(constants.SCRIPTS_DIR)
fd, local_name = tempfile.mkstemp(prefix=stage, dir=constants.SCRIPTS_DIR)
try:
util.fetchFile(script, local_name)
# check the interpreter
fh = os.fdopen(fd)
fh.seek(0)
line = fh.readline(40)
fh.close()
except:
raise RuntimeError("Unable to fetch script %s" % script)
if not line.startswith('#!'):
raise RuntimeError("Missing interpreter in %s." % script)
interp = line[2:].split()
if interp[0] == '/usr/bin/env':
if len (interp) < 2 or interp[1] not in ['python']:
raise RuntimeError("Invalid interpreter %s in %s." % (interp[1], script))
elif interp[0] not in ['/bin/sh', '/bin/bash', '/usr/bin/python']:
raise RuntimeError("Invalid interpreter %s in %s." % (interp[0], script))
cmd = [local_name]
cmd.extend(args)
os.chmod(local_name, stat.S_IRUSR | stat.S_IXUSR)
os.environ['XS_STAGE'] = stage
rc, out, err = util.runCmd2(cmd, with_stdout=True, with_stderr=True)
logger.log("Script returned %d" % rc)
# keep script, will be collected in support tarball
return rc, out, err
def log_available_disks():
disks = getQualifiedDiskList()
# make sure we have discovered at least one disk and
# at least one network interface:
if len(disks) == 0:
logger.log("No disks found on this host.")
else:
# make sure that we have enough disk space:
logger.log("Found disks: %s" % str(disks))
diskSizes = [getDiskDeviceSize(x) for x in disks]
diskSizesGB = [blockSizeToGBSize(x) for x in diskSizes]
logger.log("Disk sizes: %s" % str(diskSizesGB))
old_dom0disks = filter(
lambda x: constants.min_primary_disk_size_old <= x, diskSizesGB)
if len(old_dom0disks) == 0:
logger.log(
"Unable to find a suitable disk (with a size greater than %dGB) to install to."
% constants.min_primary_disk_size_old)
dom0disks = filter(lambda x: constants.min_primary_disk_size <= x,
diskSizesGB)
if len(dom0disks) == 0:
logger.log(
"Unable to find a suitable disk (with a size greater than %dGB) to install to."
% constants.min_primary_disk_size)
def restore_file(src_base, f, d=None):
if not d: d = f
src = os.path.join(src_base, f)
dst = os.path.join(mounts['root'], d)
if os.path.exists(src):
logger.log("Restoring /%s" % f)
util.assertDir(os.path.dirname(dst))
if os.path.isdir(src):
util.runCmd2(['cp', '-a', src, os.path.dirname(dst)])
else:
util.runCmd2(['cp', '-a', src, dst])
abs_f = os.path.join('/', f)
abs_d = os.path.join('/', d)
copy_ownership(src_base, abs_f, mounts['root'], abs_d)
for dirpath, dirnames, filenames in os.walk(src):
for i in dirnames + filenames:
src_path = os.path.join(dirpath, i)[len(src_base):]
dst_path = os.path.join(abs_d,
src_path[len(abs_f) + 1:])
copy_ownership(src_base, src_path, mounts['root'],
dst_path)
else:
logger.log("WARNING: /%s did not exist in the backup image." %
f)
def exn_error_dialog(logname, with_hd, interactive=True):
if screen:
_, exn, _ = sys.exc_info()
exn_str = str(exn)
text = constants.error_string(exn_str, logname, with_hd)
bb = ButtonBar(screen, ['Reboot'])
t = TextboxReflowed(50, text, maxHeight=screen.height - 13)
screen.pushHelpLine(" Press <Enter> to reboot.")
g = GridFormHelp(screen, "Error occurred", None, 1, 2)
g.add(t, 0, 0, padding=(0, 0, 0, 1))
g.add(bb, 0, 1, growx=1)
g.addHotKey("F2")
if not interactive:
g.setTimer(constants.AUTO_EXIT_TIMER)
result = g.runOnce()
screen.popHelpLine()
# did they press the secret F2 key that activates debugging
# features?
if result == "F2":
traceback_dialog()
else:
logger.log(
"A text UI error dialog was requested, but the UI has not been initialized yet."
)
def main(args):
ui = tui
logger.log("Starting user interface")
ui.init_ui()
status = go(ui, args, None, None)
logger.log("Shutting down user interface")
ui.end_ui()
return status
def wait_for_multipathd():
for i in range(0,120):
if mpath_cli_is_working():
return
time.sleep(1)
msg = "Unable to contact Multipathd daemon"
logger.log(msg)
raise Exception(msg)
def findRepositoriesOnMedia(drivers=False):
""" Returns a list of repositories available on local media. """
static_device_patterns = ['sd*', 'scd*', 'sr*', 'xvd*', 'nvme*n*', 'vd*']
static_devices = []
for pattern in static_device_patterns:
static_devices.extend(
map(os.path.basename, glob.glob('/sys/block/' + pattern)))
removable_devices = diskutil.getRemovableDeviceList()
removable_devices = filter(lambda x: not x.startswith('fd'),
removable_devices)
parent_devices = []
partitions = []
for dev in removable_devices + static_devices:
if os.path.exists("/dev/%s" % dev):
if os.path.exists("/sys/block/%s" % dev):
dev_partitions = diskutil.partitionsOnDisk(dev)
if len(dev_partitions) > 0:
partitions.extend(
[x for x in dev_partitions if x not in partitions])
else:
if dev not in parent_devices:
parent_devices.append(dev)
else:
if dev not in parent_devices:
parent_devices.append(dev)
da = None
repos = []
try:
for check in parent_devices + partitions:
device_path = "/dev/%s" % check
logger.log("Looking for repositories: %s" % device_path)
if os.path.exists(device_path):
da = DeviceAccessor(device_path)
try:
da.start()
except util.MountFailureException:
da = None
continue
else:
if drivers:
repo = da.findDriverRepository()
else:
repo = da.findRepository()
if repo:
repos.append(repo)
da.finish()
da = None
finally:
if da:
da.finish()
return repos
def find_installed_products():
try:
installed_products = findXenSourceProducts()
except Exception as e:
logger.log(
"A problem occurred whilst scanning for existing installations:")
logger.logException(e)
logger.log("This is not fatal. Continuing anyway.")
installed_products = []
return installed_products
def umount(mountpoint, force=False):
logger.log("Unmounting %s (force = %s)" % (mountpoint, force))
cmd = ['/bin/umount', '-d'] # -d option also removes the loop device (if present)
if force:
cmd.append('-f')
cmd.append(mountpoint)
rc = runCmd2(cmd)
return rc
def fetch(location):
logger.log("Fetching answerfile from %s" % location)
util.fetchFile(location, ANSWERFILE_PATH)
try:
xmldoc = xml.dom.minidom.parse(ANSWERFILE_PATH)
except:
raise AnswerfileException("Answerfile is incorrectly formatted.")
return Answerfile(xmldoc)
def processAnswerfileSetup(self):
"""Process enough of the answerfile so that disks can be made available
for inspection."""
logger.log("Processing XML answerfile setup.")
results = {}
results.update(self.parseDriverSource())
results.update(self.parseFCoEInterface())
results.update(self.parseUIConfirmationPrompt())
return results
def settingsAvailable(self):
try:
self.readSettings()
except SettingsNotAvailable as text:
logger.log("Settings unavailable: %s" % text)
return False
except Exception as e:
logger.log("Settings unavailable: unhandled exception")
logger.logException(e)
return False
else:
return True
def execute(self, answers):
assert type(self.predicates) == list
assert False not in [callable(x) for x in self.predicates]
assert callable(self.fn)
if False not in [x(answers) for x in self.predicates]:
logger.log("Displaying screen %s" % self.fn)
return self.fn(answers, *self.args)
else:
logger.log(
"Not displaying screen %s due to predicate return false." %
self.fn)
return SKIP_SCREEN
def isUpgradeable(self):
self.mount_state()
result = True
try:
# CA-38459: handle missing firstboot directory e.g. Rio
if os.path.exists(self.join_state_path('etc/firstboot.d/state')):
firstboot_files = [ f for f in os.listdir(self.join_state_path('etc/firstboot.d')) \
if f[0].isdigit() and os.stat(self.join_state_path('etc/firstboot.d', f))[stat.ST_MODE] & stat.S_IXUSR ]
missing_state_files = filter(
lambda x: not os.path.exists(
self.join_state_path('etc/firstboot.d/state', x)),
firstboot_files)
result = (len(missing_state_files) == 0)
if not result:
logger.log('Upgradeability test failed:')
logger.log(' Firstboot: ' +
', '.join(firstboot_files))
logger.log(' Missing state: ' +
', '.join(missing_state_files))
else:
for path in constants.INIT_SERVICE_FILES:
if not os.path.exists(self.join_state_path(path)):
result = False
logger.log(
'Cannot upgrade, expected file missing: %s' %
(path, ))
except Exception:
result = False
finally:
self.unmount_state()
return result
def main(args):
results = {}
dests = []
ui = None
xelogging.openLog('/dev/tty3')
if len(args) == 0:
ui = tui
else:
dests = args
if ui:
ui.init_ui()
results['network-hardware'] = netutil.scanConfiguration()
local_dest = lambda a: a['dest-media'] == 'local'
remote_dest = lambda a: a['dest-media'] != 'local'
seq = [
uicontroller.Step(select_report_media),
uicontroller.Step(tui.network.requireNetworking,
predicates=[remote_dest]),
uicontroller.Step(get_local_disk, predicates=[local_dest]),
uicontroller.Step(select_report_dest),
]
rc = uicontroller.runSequence(seq, results)
if rc == uicontroller.RIGHT_FORWARDS:
xelogging.log("ANSWERS DICTIONARY:")
xelogging.log(str(results))
if results['dest-media'] == 'local':
dests.append("dev://" + results['dest-address'])
elif results['dest-media'] == 'ftp':
dests.append(results['dest-address'])
elif results['dest-media'] == 'nfs':
dests.append("nfs://" + results['dest-address'])
# create tarball
collectLogs('/tmp', '/tmp')
report_saved = False
for dest in dests:
xelogging.log("Saving report to: " + dest)
try:
a = xcp.accessor.createAccessor(dest, False)
a.start()
fh = open('/tmp/support.tar.bz2')
a.writeFile(fh, 'support.tar.bz2')
fh.close()
a.finish()
report_saved = True
except Exception, e:
xelogging.log("Failed: " + str(e))
report_saved = False
def main(args):
results = {}
dests = []
ui = None
xelogging.openLog('/dev/tty3')
if len(args) == 0:
ui = tui
else:
dests = args
if ui:
ui.init_ui()
results['network-hardware'] = netutil.scanConfiguration()
local_dest = lambda a: a['dest-media'] == 'local'
remote_dest = lambda a: a['dest-media'] != 'local'
seq = [
uicontroller.Step(select_report_media),
uicontroller.Step(tui.network.requireNetworking, predicates = [remote_dest]),
uicontroller.Step(get_local_disk, predicates = [local_dest]),
uicontroller.Step(select_report_dest),
]
rc = uicontroller.runSequence(seq, results)
if rc == uicontroller.RIGHT_FORWARDS:
xelogging.log("ANSWERS DICTIONARY:")
xelogging.log(str(results))
if results['dest-media'] == 'local':
dests.append("dev://" + results['dest-address'])
elif results['dest-media'] == 'ftp':
dests.append(results['dest-address'])
elif results['dest-media'] == 'nfs':
dests.append("nfs://" + results['dest-address'])
# create tarball
collectLogs('/tmp', '/tmp')
report_saved = False
for dest in dests:
xelogging.log("Saving report to: " + dest)
try:
a = xcp.accessor.createAccessor(dest, False)
a.start()
fh = open('/tmp/support.tar.bz2')
a.writeFile(fh, 'support.tar.bz2')
fh.close()
a.finish()
report_saved = True
except Exception, e:
xelogging.log("Failed: " + str(e))
report_saved = False
def dump_ibft():
logger.log("Dump iBFT:")
for path, dirs, files in os.walk('/sys/firmware/ibft'):
for item in dirs:
logger.log(os.path.join(path, item) + '/')
for item in files:
item = os.path.join(path, item)
with open(item, 'r') as f:
data = f.read()
logger.log('%s %s' % (item, repr(data)))
logger.log("End of iBFT dump")
def processAnswerfile(self):
logger.log("Processing XML answerfile for %s." % self.operation)
if self.operation == 'installation':
install_type = getStrAttribute(self.top_node, ['mode'], default='fresh')
if install_type == "fresh":
results = self.parseFreshInstall()
elif install_type == "reinstall":
results = self.parseReinstall()
elif install_type == "upgrade":
results = self.parseUpgrade()
else:
raise AnswerfileException("Unknown mode, %s" % install_type)
results['netinstall-gpg-check'] = getBoolAttribute(self.top_node, ['netinstall-gpg-check'], default = True)
results.update(self.parseCommon())
elif self.operation == 'restore':
results = self.parseRestore()
return results
def mount(dev, mountpoint, options=None, fstype=None):
logger.log("Mounting %s to %s, options = %s, fstype = %s" % (dev, mountpoint, options, fstype))
cmd = ['/bin/mount']
if options:
assert type(options) == list
if fstype:
cmd += ['-t', fstype]
if options:
cmd += ['-o', ",".join(options)]
cmd.append(dev)
cmd.append(mountpoint)
rc, out, err = runCmd2(cmd, with_stdout=True, with_stderr=True)
if rc != 0:
raise MountFailureException("out: '%s' err: '%s'" % (out, err))
def check_repo_def(definition, require_base_repo):
""" Check that the repository source definition gives access to suitable
repositories. """
try:
tui.progress.showMessageDialog("Please wait", "Searching for repository...")
repos = repository.repositoriesFromDefinition(*definition)
tui.progress.clearModelessDialog()
except Exception as e:
logger.log("Exception trying to access repository: %s" % e)
logger.logException(e)
tui.progress.clearModelessDialog()
return REPOCHK_NO_ACCESS
else:
if len(repos) == 0:
return REPOCHK_NO_REPO
elif constants.MAIN_REPOSITORY_NAME not in [r.identifier() for r in repos] and require_base_repo:
return REPOCHK_NO_BASE_REPO
return REPOCHK_NO_ERRORS
def __init__(self, url):
self._url = url
if self._url.getScheme() not in ['http', 'https', 'ftp', 'file']:
raise Exception('Unsupported URL scheme')
if self._url.getScheme() in ['http', 'https']:
username = self._url.getUsername()
if username is not None:
logger.log("Using basic HTTP authentication")
hostname = self._url.getHostname()
password = self._url.getPassword()
self.passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
self.passman.add_password(None, hostname, username, password)
self.authhandler = urllib2.HTTPBasicAuthHandler(self.passman)
self.opener = urllib2.build_opener(self.authhandler)
urllib2.install_opener(self.opener)
logger.log("Initializing URLRepositoryAccessor with base address %s" %
str(self._url))
def runCmd2(command, with_stdout=False, with_stderr=False, inputtext=None):
cmd = subprocess.Popen(command, bufsize=1,
stdin=(inputtext and subprocess.PIPE or None),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=isinstance(command, str),
close_fds=True)
# if inputtext:
# (out, err) = cmd.communicate(inputtext)
# rv = cmd.returncode
# else:
# (stdout, stderr) = (cmd.stdout, cmd.stderr)
# for line in stdout:
# out += line
# for line in stderr:
# err += line
# rv = cmd.wait()
# the above has a deadlock condition.
# The following should suffice in all cases
(out, err) = cmd.communicate(inputtext)
rv = cmd.returncode
l = "ran %s; rc %d" % (str(command), rv)
if inputtext:
l += " with input %s" % inputtext
if out != "":
l += "\nSTANDARD OUT:\n" + out
if err != "":
l += "\nSTANDARD ERROR:\n" + err
logger.log(l)
if with_stdout and with_stderr:
return rv, out, err
elif with_stdout:
return rv, out
elif with_stderr:
return rv, err
return rv
def fetchFile(source, dest):
cleanup_dirs = []
try:
# if it's NFS, then mount the NFS server then treat like
# file://:
if source[:4] == 'nfs:':
# work out the components:
[_, server, path] = source.split(':')
if server[:2] != '//':
raise InvalidSource("Did not start {ftp,http,file,nfs}://")
server = server[2:]
dirpart = os.path.dirname(path)
if dirpart[0] != '/':
raise InvalidSource("Directory part of NFS path was not an absolute path.")
filepart = os.path.basename(path)
logger.log("Split nfs path into server: %s, directory: %s, file: %s." % (server, dirpart, filepart))
# make a mountpoint:
mntpoint = tempfile.mkdtemp(dir='/tmp', prefix='fetchfile-nfs-')
mount('%s:%s' % (server, dirpart), mntpoint, fstype="nfs", options=['ro'])
cleanup_dirs.append(mntpoint)
source = 'file://%s/%s' % (mntpoint, filepart)
if source[:5] == 'http:' or \
source[:6] == 'https:' or \
source[:5] == 'file:' or \
source[:4] == 'ftp:':
# This something that can be fetched using urllib2:
fd = urllib2.urlopen(source)
fd_dest = open(dest, 'w')
shutil.copyfileobj(fd, fd_dest)
fd_dest.close()
fd.close()
else:
raise InvalidSource("Unknown source type.")
finally:
for d in cleanup_dirs:
umount(d)
os.rmdir(d)
def installKeys(self, root):
if len(self.keyfiles) == 0:
return
keysdir = os.path.join(root, 'etc', 'firstboot.d', 'data', 'keys')
if not os.path.exists(keysdir):
os.makedirs(keysdir, 0755)
self._accessor.start()
try:
for keyfile in self.keyfiles:
infh = self._accessor.openAddress(keyfile)
outfh = open(os.path.join(keysdir, os.path.basename(keyfile)),
"w")
outfh.write(infh.read())
outfh.close()
infh.close()
except Exception as e:
logger.log(str(e))
self._accessor.finish()
raise UnrecoverableRepoError("Error installing key files")
self._accessor.finish()
def findXenSourceProducts():
"""Scans the host and finds XenSource product installations.
Returns list of ExistingInstallation objects.
Currently requires supervisor privileges due to mounting
filesystems."""
installs = []
for disk in diskutil.getQualifiedDiskList():
(boot, root, state, storage, logs) = diskutil.probeDisk(disk)
inst = None
try:
if root[0] == diskutil.INSTALL_RETAIL:
inst = ExistingRetailInstallation(disk, boot[1], root[1],
state[1], storage)
except Exception as e:
logger.log(
"A problem occurred whilst scanning for existing installations:"
)
logger.logException(e)
logger.log("This is not fatal. Continuing anyway.")
if inst:
logger.log("Found an installation: %s on %s" % (str(inst), disk))
installs.append(inst)
return installs
def mpath_enable():
global use_mpath
assert 0 == util.runCmd2(['modprobe','dm-multipath'])
if not os.path.exists('/etc/multipath.conf') and os.path.exists('/etc/multipath.conf.disabled'):
os.rename('/etc/multipath.conf.disabled', '/etc/multipath.conf')
# launch manually to make possible to wait initialization
util.runCmd2(["/sbin/multipath", "-v0", "-B"])
time.sleep(1)
util.runCmd2(util.udevsettleCmd())
# This creates maps for all disks at start of day (because -e is ommitted)
assert 0 == util.runCmd2('multipathd -d > /var/log/multipathd 2>&1 &')
wait_for_multipathd()
# CA-48440: Cope with lost udev events
util.runCmd2(["multipathd","-k"], inputtext="reconfigure")
# Tell DM to create partition nodes for newly created mpath devices
assert 0 == mpath_part_scan(True)
logger.log("created multipath device(s)");
use_mpath = True
def setup_ibft_nics():
mac_map = {}
netdevs = netutil.scanConfiguration()
for name in netdevs:
mac_map[netdevs[name].hwaddr] = name
logger.log('NET: %s %s' % (repr(netdevs), repr(mac_map)))
for t in glob.glob(os.path.join(constants.SYSFS_IBFT_DIR, 'target*')):
with open(os.path.join(t, 'ip-addr'), 'r') as f:
target_ip = f.read().strip()
with open(os.path.join(t, 'nic-assoc'), 'r') as f:
nic_assoc = f.read().strip()
e = os.path.join(constants.SYSFS_IBFT_DIR, 'ethernet' + nic_assoc)
with open(os.path.join(e, 'mac'), 'r') as f:
mac = f.read().strip()
with open(os.path.join(e, 'ip-addr'), 'r') as f:
ip = f.read().strip()
try:
with open(os.path.join(e, 'gateway'), 'r') as f:
gw = f.read().strip()
except IOError as err:
if err.errno == errno.ENOENT:
gw = None
else:
raise
with open(os.path.join(e, 'subnet-mask'), 'r') as f:
nm = f.read().strip()
with open(os.path.join(e, 'flags'), 'r') as f:
flags = int(f.read().strip())
assert (flags & 3) == 3
if mac not in mac_map:
raise RuntimeError(
'Found mac %s in iBFT but cannot find matching NIC' % mac)
configure_ibft_nic(target_ip, mac_map[mac], ip, nm, gw)
ibft_reserved_nics.add(mac_map[mac])
def parseExistingInstallation(self):
results = {}
inst = getElementsByTagName(self.top_node, ['existing-installation'],
mandatory=True)
disk = normalize_disk(getText(inst[0]))
logger.log("Normalized disk: %s" % disk)
disk = disktools.getMpathMasterOrDisk(disk)
logger.log('Primary disk: ' + disk)
results['primary-disk'] = disk
installations = product.findXenSourceProducts()
installations = filter(lambda x: x.primary_disk == disk or diskutil.idFromPartition(x.primary_disk) == disk, installations)
if len(installations) == 0:
raise AnswerfileException("Could not locate the installation specified to be reinstalled.")
elif len(installations) > 1:
# FIXME non-multipath case?
logger.log("Warning: multiple paths detected - recommend use of --device_mapper_multipath=yes")
logger.log("Warning: selecting 1st path from %s" % str(map(lambda x: x.primary_disk, installations)))
results['installation-to-overwrite'] = installations[0]
return results
