def _rootfs_guess(target, image, boot_dev):
reason = "unknown issue"
for tries in range(3):
tries += 1
try:
target.report_info("POS: guessing partition device [%d/3]" % tries)
root_part_dev = _rootfs_guess_by_image(target, image, boot_dev)
if root_part_dev != None:
return root_part_dev
# we couldn't find a root partition device, which means the
# thing is trashed
target.report_info("POS: repartitioning because couldn't find "
"root partitions")
_disk_partition(target)
target.pos._fsinfo_load()
except Exception as e:
reason = str(e)
if tries < 3:
target.report_info("POS: failed to guess a root partition, "
"retrying: %s" % reason)
continue
else:
raise
raise tc.blocked_e(
"Tried too much to reinitialize the partition table to "
"pick up a root partition? is there enough space to "
"create root partitions?",
dict(target = target, reason = reason,
partsizes = target.kws.get('pos_partsizes', None)))
def _rootfs_guess(target, image, boot_dev):
reason = "unknown issue"
for tries in range(3):
tries += 1
try:
target.report_info("POS: guessing partition device [%d/3]" % tries)
root_part_dev = _rootfs_guess_by_image(target, image, boot_dev)
if root_part_dev != None:
return root_part_dev
# we couldn't find a root partition device, which means the
# thing is trashed
target.report_info("POS: repartitioning because couldn't find "
"root partitions")
_disk_partition(target)
target.pos.fsinfo_read(target._boot_label_name)
except Exception as e:
reason = str(e)
if tries < 3:
target.report_info("POS: failed to guess a root partition, "
"retrying: %s" % reason)
continue
else:
raise
raise tc.blocked_e(
"Tried too much to reinitialize the partition table to "
"pick up a root partition? is there enough space to "
"create root partitions?",
dict(target=target,
reason=reason,
partsizes=target.kws.get('pos_partsizes', None)))
def _linux_boot_guess_from_boot(target, image):
"""
Given a list of files (normally) in /boot, decide which ones are
Linux kernels and initramfs; select the latest version
"""
# guess on the mounted filesystem, otherwise we get the POS!
os_release = tl.linux_os_release_get(target, prefix="/mnt")
distro = os_release.get('ID', None)
output = target.shell.run("ls -1 /mnt/boot", output=True)
kernel_regex = re.compile("(initramfs|initrd|bzImage|vmlinuz)(-(.*))?")
kernel_versions = {}
initramfs_versions = {}
for line in output.split('\n'):
m = kernel_regex.match(line)
if not m:
continue
file_name = m.groups()[0]
kver = m.groups()[1]
if kver == None:
kver = "default"
if kver and ("rescue" in kver or "kdump" in kver):
# these are usually found on Fedora
continue
elif file_name in ("initramfs", "initrd"):
if kver.endswith(".img"):
# remove .img extension that has been pegged to the version
kver = os.path.splitext(kver)[0]
initramfs_versions[kver] = line
else:
kernel_versions[kver] = line
if len(kernel_versions) > 1 and 'default' in kernel_versions:
del kernel_versions['default']
if len(kernel_versions) == 1:
kver = kernel_versions.keys()[0]
options = ""
# image is atuple of (DISTRO, SPIN, VERSION, SUBVERSION, ARCH)
if distro in ("fedora", "debian", "ubuntu") and 'live' in image:
# Live distros needs this to boot, unknown exactly why;
# also add console=tty0 to ensure it is not lost
target.report_info("Linux Live hack: adding 'rw' to cmdline",
dlevel=2)
options = "console=tty0 rw"
kernel = kernel_versions[kver]
if kernel:
kernel = "/boot/" + kernel
initrd = initramfs_versions.get(kver, None)
if initrd:
initrd = "/boot/" + initrd
return kernel, initrd, options
elif len(kernel_versions) > 1:
raise tc.blocked_e(
"more than one Linux kernel in /boot; I don't know "
"which one to use: " + " ".join(kernel_versions),
dict(target=target, output=output))
else:
return None, None, ""
def _linux_boot_guess_from_boot(target, image):
"""
Given a list of files (normally) in /boot, decide which ones are
Linux kernels and initramfs; select the latest version
"""
# guess on the mounted filesystem, otherwise we get the POS!
os_release = tl.linux_os_release_get(target, prefix = "/mnt")
distro = os_release.get('ID', None)
output = target.shell.run("ls -1 /mnt/boot", output = True)
kernel_regex = re.compile("(initramfs|initrd|bzImage|vmlinuz)(-(.*))?")
kernel_versions = {}
initramfs_versions = {}
for line in output.split('\n'):
m = kernel_regex.match(line)
if not m:
continue
file_name = m.groups()[0]
kver = m.groups()[1]
if kver == None:
kver = "default"
if kver and ("rescue" in kver or "kdump" in kver):
# these are usually found on Fedora
continue
elif file_name in ( "initramfs", "initrd" ):
if kver.endswith(".img"):
# remove .img extension that has been pegged to the version
kver = os.path.splitext(kver)[0]
initramfs_versions[kver] = line
else:
kernel_versions[kver] = line
if len(kernel_versions) > 1 and 'default' in kernel_versions:
del kernel_versions['default']
if len(kernel_versions) == 1:
kver = kernel_versions.keys()[0]
options = ""
# image is atuple of (DISTRO, SPIN, VERSION, SUBVERSION, ARCH)
if distro in ("fedora", "debian", "ubuntu") and 'live' in image:
# Live distros needs this to boot, unknown exactly why;
# also add console=tty0 to ensure it is not lost
target.report_info("Linux Live hack: adding 'rw' to cmdline",
dlevel = 2)
options = "console=tty0 rw"
kernel = kernel_versions[kver]
if kernel:
kernel = "/boot/" + kernel
initrd = initramfs_versions.get(kver, None)
if initrd:
initrd = "/boot/" + initrd
return kernel, initrd, options
elif len(kernel_versions) > 1:
raise tc.blocked_e(
"more than one Linux kernel in /boot; I don't know "
"which one to use: " + " ".join(kernel_versions),
dict(target = target, output = output))
else:
return None, None, ""
def console_rx_poller(expecter, target, console = None):
"""
Poll a console
"""
# Figure out to which file we are writing
console_id_name, console_code = console_mk_code(target, console)
of = expecter.buffers.setdefault(
console_code,
open(os.path.join(target.testcase.buffersdir,
"console-%s:%s-%s.log" % (
commonl.file_name_make_safe(target.fullid),
target.kws['tc_hash'], console_id_name)),
"a+", 0))
ofd = of.fileno()
expecter.buffers.setdefault(console_code + "-ts0", time.time())
# Don't read too much, leave the rest for another run -- otherwise
# we could spend our time trying to read a 1G console log file
# from a really broken test case that spews a lot of stuff.
# FIXME: move this to configuration
max_size = 3000
# Read anything new since the last time we read -- this relies
# on we having an exclusive lock on the target
try:
offset = os.fstat(ofd).st_size
ts_start = time.time()
target.report_info("reading from console %s @%d at %.2fs [%s]"
% (console_id_name, offset,
ts_start - expecter.ts0, of.name),
dlevel = 3)
# We are dealing with a file as our buffering and accounting
# system, so because read_to_fd() is bypassing caching, flush
# first and sync after the read.
of.flush()
total_bytes = target.rtb.rest_tb_target_console_read_to_fd(
ofd, target.rt, console, offset, max_size, target.ticket)
of.flush()
os.fsync(ofd)
ts_end = time.time()
target.report_info("read from console %s @%d %dB at %.2fs (%.2fs) "
"[%s]"
% (console_id_name, offset, total_bytes,
ts_end - expecter.ts0, ts_end - ts_start,
of.name),
dlevel = 3)
# FIXME: do we want to print some debug of what we read? how
# do we do it for huge files anyway?
expecter._consoles.add(( target, console ))
except requests.exceptions.HTTPError as e:
raise tc.blocked_e("error reading console %s: %s\n"
% (console_id_name, e),
{ "error trace": traceback.format_exc() })
# Don't count this as something that we need to treat as succesful
return None
def console_rx_poller(expecter, target, console=None):
"""
Poll a console
"""
# Figure out to which file we are writing
console_id_name, console_code = console_mk_code(target, console)
of = expecter.buffers.setdefault(
console_code,
open(
os.path.join(
target.testcase.buffersdir,
"console-%s:%s-%s.log" % (commonl.file_name_make_safe(
target.fullid), target.kws['tc_hash'], console_id_name)),
"a+", 0))
ofd = of.fileno()
expecter.buffers.setdefault(console_code + "-ts0", time.time())
# Don't read too much, leave the rest for another run -- otherwise
# we could spend our time trying to read a 1G console log file
# from a really broken test case that spews a lot of stuff.
# FIXME: move this to configuration
max_size = 3000
# Read anything new since the last time we read -- this relies
# on we having an exclusive lock on the target
try:
offset = os.fstat(ofd).st_size
ts_start = time.time()
target.report_info(
"reading from console %s @%d at %.2fs [%s]" %
(console_id_name, offset, ts_start - expecter.ts0, of.name),
dlevel=3)
# We are dealing with a file as our buffering and accounting
# system, so because read_to_fd() is bypassing caching, flush
# first and sync after the read.
of.flush()
total_bytes = target.rtb.rest_tb_target_console_read_to_fd(
ofd, target.rt, console, offset, max_size, target.ticket)
of.flush()
os.fsync(ofd)
ts_end = time.time()
target.report_info("read from console %s @%d %dB at %.2fs (%.2fs) "
"[%s]" %
(console_id_name, offset, total_bytes,
ts_end - expecter.ts0, ts_end - ts_start, of.name),
dlevel=3)
# FIXME: do we want to print some debug of what we read? how
# do we do it for huge files anyway?
expecter._consoles.add((target, console))
except requests.exceptions.HTTPError as e:
raise tc.blocked_e(
"error reading console %s: %s\n" % (console_id_name, e),
{"error trace": traceback.format_exc()})
# Don't count this as something that we need to treat as succesful
return None
def kbd_string_send(self,
string,
did="default_keyboard",
press_time=0.2,
interkey_time=0.2):
"""
Send a sequence of strings to a keyboard as key presses
:param str string: string to send; note some characters can't
be sent as is and have to be translated to key sequences
(eg: press and hold shift, key X, relase shift). This is
still not supported (FIXME)
:param str did: (optional) name of the device to which to send it
:param float press_time: (optional) time in seconds to press
the key
:param float interkey_time: (optional) time in seconds to wait
between pressing keys for each string character
"""
assert isinstance(string, basestring)
assert isinstance(did, basestring)
assert isinstance(press_time, numbers.Real)
assert isinstance(interkey_time, numbers.Real)
device = self._device_get(did)
s = ""
for char in string.lower():
if char in self.key_mapping:
key_code = self.key_mapping[char]
else:
raise tc.blocked_e(
"still don't know how to send character '%s', add to"
"input.key_mapping" % char)
s += \
"%s EV_KEY %s 1\n" \
"WAIT %s\n" \
"%s EV_KEY %s 0 SYNC\n" \
"WAIT %s\n" \
% (device, key_code, press_time,
device, key_code, interkey_time)
self.sequence_send(s)
def _disk_partition(target):
# we assume we are going to work on the boot device
device_basename = target.kws['pos_boot_dev']
device = "/dev/" + device_basename
target.shell.run('swapoff -a || true') # in case we autoswapped
# find device size (FIXME: Linux specific)
dev_info = None
for blockdevice in target.pos.fsinfo.get('blockdevices', []):
if blockdevice['name'] == device_basename:
dev_info = blockdevice
break
else:
raise tc.error_e(
"%s: can't find information about this block device -- is "
"the right pos_boot_device set in the configuration?"
% device_basename,
dict(fsinfo = pprint.pformat(target.pos.fsinfo)))
size_gb = int(dev_info['size']) / 1024 / 1024 / 1024
target.report_info("POS: %s is %d GiB in size" % (device, size_gb),
dlevel = 2)
partsizes = target.kws.get('pos_partsizes', None)
if partsizes == None:
raise tc.blocked_e(
"Can't partition target, it doesn't "
"specify pos_partsizes tag",
{ 'target': target } )
partsize_l = partsizes.split(":")
partsize_l = [ int(_partsize) for _partsize in partsize_l ]
boot_size = partsize_l[0]
swap_size = partsize_l[1]
scratch_size = partsize_l[2]
root_size = partsize_l[3]
# note we set partition #0 as boot
cmdline = """parted -a optimal -ms %(device)s unit GiB \
mklabel gpt \
mkpart primary fat32 0%% %(boot_size)s \
set 1 boot on \
mkpart primary linux-swap %(boot_size)s %(swap_end)s \
mkpart primary ext4 %(swap_end)s %(scratch_end)s \
""" % dict(
device = device,
boot_size = boot_size,
swap_end = boot_size + swap_size,
scratch_end = boot_size + swap_size + scratch_size,
)
offset = boot_size + swap_size + scratch_size
root_devs = [] # collect the root devices
pid = 4
while offset + root_size < size_gb:
cmdline += ' mkpart primary ext4 %d %d' % (offset, offset + root_size)
offset += root_size
root_devs.append(device_basename + target.kws['p_prefix']
+ "%d" % pid)
pid += 1
target.shell.run(cmdline)
# Now set the root device information, so we can pick stuff to
# format quick
for root_dev in root_devs:
target.property_set('pos_root_' + root_dev, "EMPTY")
# Re-read partition tables
target.shell.run('partprobe %s' % device)
# now format filesystems
#
# note we only format the system boot partition (1), the linux
# swap(2) and the linux scratch space (3)
boot_dev = device + target.kws['p_prefix'] + "1"
swap_dev = device + target.kws['p_prefix'] + "2"
home_dev = device + target.kws['p_prefix'] + "3"
# Note: use FAT vs VFAT: vfat name translation creates issues when
# doing long file names; fat32 does not have that problem.
target.shell.run("mkfs.fat -F32 -n TCF-BOOT " + boot_dev)
target.shell.run("mkswap -L tcf-swap " + swap_dev)
target.shell.run("mkfs.ext4 -FqL tcf-scratch " + home_dev)
def flash(self,
images,
upload=True,
timeout=None,
soft=False,
hash_target_name=True):
"""Flash images onto target
>>> target.images.flash({
>>> "kernel-86": "/tmp/file.bin",
>>> "kernel-arc": "/tmp/file2.bin"
>>> }, upload = True)
or:
>>> target.images.flash({
>>> "vmlinuz": "/tmp/vmlinuz",
>>> "initrd": "/tmp/initrd"
>>> }, upload = True)
If *upload* is set to true, this function will first upload
the images to the server and then flash them.
:param dict images: dictionary keyed by (str) image type of
things to flash in the target. e.g.:
The types if images supported are determined by the target's
configuration and can be reported with :meth:`list` (or
command line *tcf images-ls TARGETNAME*).
:param int timeout: (optional) seconds to wait for the
operation to complete; defaults to whatever the interface
declares in property
*interfaces.images.IMAGETYPE.estimated_duration*.
This is very tool and file specific, a bigger file with a
slow tool is going to take way longer than a bigfer file on
a slow one.
:param bool upload: (optional) the image names are local files
that need to be uploaded first to the server (this function
will take care of that).
:param boot soft: (optional, default *False*) if *True*, it
will only flash an image if the hash of the file is
different to the hash of the last image recorded in that
image type (or if there is no record of anything having been
flashed).
"""
if isinstance(images, dict):
for k, v in images.items():
assert isinstance(k, basestring) \
and isinstance(v, basestring), \
"images has to be a dictionary IMAGETYPE:IMAGEFILE;" \
" all strings; %s, %s (%s, %s)" \
% (k, v, type(k), type(v))
else:
raise AssertionError(
"images has to be a dictionary IMAGETYPE:IMAGEFILE; got %s" \
% type(images))
target = self.target
images_str = " ".join("%s:%s" % (k, v) for k, v in images.items())
if timeout == None:
timeout = 0
for image_type, image in images.items():
images_data = target.rt['interfaces']['images']
image_data = images_data.get(image_type, None)
if image_data == None:
raise tc.blocked_e(
"%s: image type '%s' not available" %
(target.id, image_type), dict(target=target))
timeout += image_data.get("estimated_duration", 60)
else:
assert isinstance(timeout, int)
# if we have to upload them, then we'll transform the names to
# point to the names we got when uploading
if upload:
# Ok, we need to upload--the names in the dictionary point
# to local filenames relative to the dir where we are
# from, or absolute. Upload them to the server file space
# for the user and give them a local name in there.
_images = {}
target.report_info("uploading: " + images_str, dlevel=2)
for img_type, img_name in images.iteritems():
# the remote name will be NAME-DIGEST, so if multiple
# testcases for the same user are uploading files with
# the same name but different content / target, they don't
# collide
hd = commonl.hash_file_maybe_compressed(
hashlib.sha512(), img_name)
img_name_remote = \
hd[:10] \
+ "-" + commonl.file_name_make_safe(os.path.abspath(img_name))
if hash_target_name:
# put the target name first, otherwise we might
# alter the extension that the server relies on to
# autodecompress if need to
img_name_remote = target.id + "-" + img_name_remote
last_sha512 = target.rt['interfaces']['images']\
[img_type].get('last_sha512', None)
if soft and last_sha512 == hd:
# soft mode -- don't flash again if the last thing
# flashed has the same hash as what we want to flash
target.report_info(
"%s:%s: skipping (soft flash: SHA512 match %s)" %
(img_type, img_name, hd),
dlevel=1)
continue
target.report_info("uploading: %s %s" % (img_type, img_name),
dlevel=3)
target.store.upload(img_name_remote, img_name)
_images[img_type] = img_name_remote
target.report_info("uploaded: %s %s" % (img_type, img_name),
dlevel=2)
target.report_info("uploaded: " + images_str, dlevel=1)
else:
_images = images
if _images:
# We don't do retries here, we leave it to the server
target.report_info("flashing: " + images_str, dlevel=2)
target.ttbd_iface_call("images",
"flash",
images=_images,
timeout=timeout)
target.report_info("flashed: " + images_str, dlevel=1)
else:
target.report_info("flash: all images soft flashed", dlevel=1)
def console_rx_eval(expecter, target,
regex, console = None, _timeout = None, result = None,
uid = None):
"""
Check what came on a console and act on it
:param str uid: (optional) identifier to use to store offset data
"""
if hasattr(regex, "pattern"):
what = regex.pattern
else:
what = regex
regex = re.compile(re.escape(regex))
if not uid:
uid = console_mk_uid(target, what, console, _timeout, result)
console_id_name, console_code = console_mk_code(target, console)
# These were set by the poller
of = expecter.buffers.get(console_code, None)
if of == None:
# FIXME: debug lof output here? expecter->tc backlink?
return None
ofd = of.fileno()
ts = time.time()
# Get the offset we saved before as the last part where we looked
# at. If none, then get the last offset the poller has
# recorded. Otherwise, just default to look from the start
# Note the idea is each eval function has a different offset where
# it is looking at. Likewise the poller for each console.
offset_poller_code = "offset_" + console_code
offset = expecter.buffers_persistent.get(
uid,
expecter.buffers_persistent.get(offset_poller_code, 0))
if _timeout != False:
timeout = expecter.timeout if _timeout == None else _timeout
# We can do timeout checks that provide better information
# than a generic 'timeout'
if ts - expecter.ts0 > timeout:
of.seek(offset) # so we report console from where searched
raise tc.error_e(
"expected console output '%s' from console '%s:%s' " \
"NOT FOUND after %.1f s" \
% (what, target.id, console_id_name, ts - expecter.ts0),
{ 'target': target, "console output": of })
# mmap the whole file (which doesn't alter the file pointer)
#
# We have to mmap as the file might be getting huge and thus,
# reading line by line might be dumb.
#
# However, we only search starting at @offset, which is set later
# to the last success searching we had. So we shan't really map
# the whole file, shall map on demand.
stat_info = os.fstat(ofd)
if stat_info.st_size == 0: # Nothing to read
return None
with contextlib.closing(mmap.mmap(of.fileno(), 0, mmap.MAP_PRIVATE,
mmap.PROT_READ, 0)) as mapping:
target.report_info("looking for `%s` in console %s:%s @%d-%d at "
"%.2fs [%s]"
% (what, target.fullid, console_id_name, offset,
stat_info.st_size, ts - expecter.ts0, of.name),
dlevel = 3)
m = regex.search(mapping[offset:])
if m:
new_offset = offset + m.end()
expecter.buffers_persistent[uid] = new_offset
if result == None or result == "pass":
# raising pass gets stopped at expecter.run(), so we
# print instead, so we can see the console
offset_tip = of.tell() # we report console from where searched
of.seek(offset) # so we report console from where searched
target.report_pass(
"found expected `%s` in console `%s:%s` at %.2fs"
% (what, target.fullid, console_id_name,
ts - expecter.ts0),
{ "console output": of }, dlevel = 1, alevel = 2)
of.seek(offset_tip)
raise tc.pass_e(
"found expected `%s` in console `%s:%s` at %.2fs"
% (what, target.fullid, console_id_name,
ts - expecter.ts0),
{'target': target })
elif result == "fail":
of.seek(offset) # so we report console from where searched
raise tc.failed_e(
"found expected (for failure) `%s` in console "
"`%s:%s` at %.2fs"
% (what, target.fullid, console_id_name,
ts - expecter.ts0),
{ 'target': target, "console output": of })
elif result == "error" or result == "errr":
of.seek(offset) # so we report console from where searched
raise tc.error_e(
"found expected (for error) `%s` in console "
"`%s:%s` at %.2fs"
% (what, target.fullid, console_id_name,
ts - expecter.ts0),
{ 'target': target, "console output": of })
elif result == "skip":
of.seek(offset) # so we report console from where searched
raise tc.skip_e(
"found expected (for skip) `%s` in console "
"'%s:%s' at %.2fs"
% (what, target.fullid, console_id_name,
ts - expecter.ts0),
{ 'target': target, "console output": of })
else:
of.seek(offset) # so we report console from where searched
raise tc.blocked_e(
"BUG: invalid result requested (%s)" % result)
return None
def mount_fs(target, image, boot_dev):
"""
Boots a root filesystem on /mnt
The partition used as a root filesystem is picked up based on the
image that is going to be installed; we look for one that has the
most similar image already installed and pick that.
:returns: name of the root partition device
"""
pos_reinitialize = target.property_get("pos_reinitialize", False)
if pos_reinitialize:
# Need to reinit the partition table (we were told to by
# setting pos_repartition to anything
target.report_info("POS: repartitioning per pos_reinitialize "
"property")
for tag in target.rt.keys():
# remove pos_root_*, as they don't apply anymore
if tag.startswith("pos_root_"):
target.property_set(tag, None)
_disk_partition(target)
target.property_set('pos_reinitialize', None)
root_part_dev = _rootfs_guess(target, image, boot_dev)
# save for other functions called later
target.root_part_dev = root_part_dev
root_part_dev_base = os.path.basename(root_part_dev)
image_prev = target.property_get("pos_root_" + root_part_dev_base,
"nothing")
target.report_info("POS: will use %s for root partition (had %s before)"
% (root_part_dev, image_prev))
# fsinfo looks like described in target.pos._fsinfo_load()
dev_info = None
for blockdevice in target.pos.fsinfo.get('blockdevices', []):
for child in blockdevice.get('children', []):
if child['name'] == root_part_dev_base:
dev_info = child
if dev_info == None:
# it cannot be we might have to repartition because at this
# point *we* have partitoned.
raise tc.error_e(
"Can't find information for root device %s in FSinfo array"
% root_part_dev_base,
dict(fsinfo = target.pos.fsinfo))
# what format does it currently have?
current_fstype = dev_info.get('fstype', 'ext4')
# What format does it have to have?
#
# Ok, here we need to note that we can't have multiple root
# filesystems with the same UUID or LABEL, so the image can't rely
# on UUIDs
#
img_fss = target.pos.metadata.get('filesystems', {})
if '/' in img_fss:
# a common origin is ok because the YAML schema forces both
# fstype and mkfs_opts to be specified
origin = "image's /.tcf.metadata.yaml"
fsdata = img_fss.get('/', {})
else:
origin = "defaults @" + commonl.origin_get(0)
fsdata = {}
fstype = fsdata.get('fstype', 'ext4')
mkfs_opts = fsdata.get('mkfs_opts', '-Fj')
# do they match?
if fstype != current_fstype:
target.report_info(
"POS: reformatting %s because current format is '%s' and "
"'%s' is needed (per %s)"
% (root_part_dev, current_fstype, fstype, origin))
_mkfs(target, root_part_dev, fstype, mkfs_opts)
else:
target.report_info(
"POS: no need to reformat %s because current format is '%s' and "
"'%s' is needed (per %s)"
% (root_part_dev, current_fstype, fstype, origin), dlevel = 1)
for try_count in range(3):
target.report_info("POS: mounting root partition %s onto /mnt "
"to image [%d/3]" % (root_part_dev, try_count))
# don't let it fail or it will raise an exception, so we
# print FAILED in that case to look for stuff; note the
# double apostrophe trick so the regex finder doens't trip
# on the command
output = target.shell.run(
"mount %s /mnt || echo FAI''LED" % root_part_dev,
output = True)
# What did we get?
if 'FAILED' in output:
if 'special device ' + root_part_dev \
+ ' does not exist.' in output:
_disk_partition(target)
else:
# ok, this probably means probably the partitions are not
# formatted; so let's just reformat and retry
_mkfs(target, root_part_dev, fstype, mkfs_opts)
else:
target.report_info("POS: mounted %s onto /mnt to image"
% root_part_dev)
return root_part_dev # it worked, we are done
# fall through, retry
else:
raise tc.blocked_e(
"POS: Tried to mount too many times and failed",
dict(target = target))
def boot_config_multiroot(target, boot_dev, image):
"""
Configure the target to boot using the multiroot
"""
boot_dev = target.kws['pos_boot_dev']
# were we have mounted the root partition
root_dir = "/mnt"
linux_kernel_file, linux_initrd_file, linux_options = \
_linux_boot_guess(target, image)
if linux_kernel_file == None:
raise tc.blocked_e(
"Cannot guess a Linux kernel to boot",
dict(target = target))
# remove absolutization (some specs have it), as we need to copy from
# mounted filesystems
if os.path.isabs(linux_kernel_file):
linux_kernel_file = linux_kernel_file[1:]
if linux_initrd_file and os.path.isabs(linux_initrd_file):
linux_initrd_file = linux_initrd_file[1:]
if linux_options == None or linux_options == "":
target.report_info("WARNING! can't figure out Linux cmdline "
"options, taking defaults")
# below we'll add more stuff
linux_options = "console=tty0 root=SOMEWHERE"
# MULTIROOT: indicate which image has been flashed to this
# partition
# saved by pos_multiroot.mountfs
root_part_dev = target.root_part_dev
root_part_dev_base = os.path.basename(root_part_dev)
target.property_set('pos_root_' + root_part_dev_base, image)
# /boot EFI system partition is always /dev/DEVNAME1 (first
# partition), we partition like that
# FIXME: we shouldn't harcode this
boot_part_dev = boot_dev + target.kws['p_prefix'] + "1"
kws = dict(
boot_dev = boot_dev,
boot_part_dev = boot_part_dev,
root_part_dev = root_part_dev,
root_part_dev_base = root_part_dev_base,
root_dir = root_dir,
linux_kernel_file = linux_kernel_file,
linux_kernel_file_basename = os.path.basename(linux_kernel_file),
linux_initrd_file = linux_initrd_file,
linux_options = linux_options,
)
if linux_initrd_file:
kws['linux_initrd_file_basename'] = os.path.basename(linux_initrd_file)
else:
kws['linux_initrd_file_basename'] = None
kws.update(target.kws)
if linux_options:
#
# Maybe mess with the Linux boot options
#
target.report_info("linux cmdline options: %s" % linux_options)
# FIXME: can this come from config?
linux_options_replace = {
# we want to use hard device name rather than LABELS/UUIDs, as
# we have reformated and those will have changed
"root": "/dev/%(root_part_dev_base)s" % kws,
# we have created this in pos_multiroot and will only
# replace it if the command line option is present.
"resume": "/dev/disk/by-label/tcf-swap",
}
# FIXME: can this come from config?
# We harcode a serial console on the device where we know the
# framework is listening
linux_options_append = \
"console=%(linux_serial_console_default)s,115200n8" % kws
linux_options_append += " " + target.rt.get('linux_options_append', "")
linux_options += " " + linux_options_append
for option, value in linux_options_replace.iteritems():
regex = re.compile(r"\b" + option + r"=\S+")
if regex.search(linux_options):
linux_options = re.sub(
regex,
option + "=" + linux_options_replace[option],
linux_options)
else:
linux_options += " " + option + "=" + value
kws['linux_options'] = linux_options
target.report_info("linux cmdline options (modified): %s"
% linux_options)
# Now generate the UEFI system partition that will boot the
# system; we always override it, so we don't need to decide if it
# is corrupted or whatever; we'll mount it in /boot (which now is
# the POS /boot)
#
# Mount the /boot fs
#
# Try to assume it is ok, try to repair it if not; rsync the
# kernels in there, it is faster for repeated operation/
#
target.report_info("POS/EFI: checking %(boot_part_dev)s" % kws)
output = target.shell.run(
"fsck.fat -aw /dev/%(boot_part_dev)s || true" % kws,
output = True, trim = True)
# FIXME: parse the output to tell if there was a problem; when bad
# but recovered, we'll see
#
# 0x41: Dirty bit is set. Fs was not properly unmounted and some data may be corrupt.
# Automatically removing dirty bit.
# Performing changes.
# /dev/sda1: 11 files, 4173/261372 clusters
# When ok
#
## $ sudo fsck.vfat -wa /dev/nvme0n1p1
## fsck.fat 4.1 (2017-01-24)
## /dev/sda1: 39 files, 2271/33259 clusters
# When really hosed it won't print the device line, so we look for
# that
#
## $ fsck.vfat -wa /dev/trash
## fsck.fat 4.1 (2017-01-24)
## Logical sector size (49294 bytes) is not a multiple of the physical sector size.
good_regex = re.compile("^/dev/%(boot_part_dev)s: [0-9]+ files, "
"[0-9]+/[0-9]+ clusters$" % kws, re.MULTILINE)
if not good_regex.search(output):
target.report_info(
"POS/EFI: /dev/%(boot_part_dev)s: formatting EFI "
"filesystem, fsck couldn't fix it"
% kws, dict(output = output))
target.shell.run("mkfs.fat -F32 /dev/%(boot_part_dev)s; sync" % kws)
target.report_info(
"POS/EFI: /dev/%(boot_part_dev)s: mounting in /boot" % kws)
target.shell.run(" mount /dev/%(boot_part_dev)s /boot; "
" mkdir -p /boot/loader/entries " % kws)
# Do we have enough space? if not, remove the oldest stuff that is
# not the file we are looking for
# This prints
## $ df --output=pcent /boot
## Use%
## 6%
output = target.shell.run("df --output=pcent /boot", output = True)
regex = re.compile(r"^\s*(?P<percent>[\.0-9]+)%$", re.MULTILINE)
match = regex.search(output)
if not match:
raise tc.error_e("Can't determine the amount of free space in /boot",
dict(output = output))
used_space = float(match.groupdict()['percent'])
if used_space > 75:
target.report_info(
"POS/EFI: /dev/%(boot_part_dev)s: freeing up space" % kws)
# List files in /boot, sort by last update (when we rsynced
# them)
## 2018-10-29+08:48:48.0000000000 84590 /boot/EFI/BOOT/BOOTX64.EFI
## 2018-10-29+08:48:48.0000000000 84590 /boot/EFI/systemd/systemd-bootx64.efi
## 2019-05-14+13:25:06.0000000000 7192832 /boot/vmlinuz-4.12.14-110-default
## 2019-05-14+13:25:08.0000000000 9688340 /boot/initrd-4.12.14-110-default
## 2019-05-14+13:25:14.0000000000 224 /boot/loader/entries/tcf-boot.conf
## 2019-05-14+13:25:14.0000000000 54 /boot/loader/loader.conf
output = target.shell.run(
# that double \\ needed so the shell is the one using it
# as a \t, not python converting to a sequence
"find /boot/ -type f -printf '%T+\\t%s\\t%p\\n' | sort",
output = True, trim = True)
# delete the first half entries over 300k except those that
# match the kernels we are installing
to_remove = []
_linux_initrd_file = kws.get("linux_initrd_file", "%%NONE%%")
if linux_initrd_file == None:
_linux_initrd_file = "%%NONE%%"
for line in output.splitlines():
_timestamp, size_s, path = line.split(None, 3)
size = int(size_s)
if size > 300 * 1024 \
and not kws['linux_kernel_file_basename'] in path \
and not _linux_initrd_file in path:
to_remove.append(path)
# get older half and wipe it--this means the first half, as we
# sort from older to newer
to_remove = to_remove[:len(to_remove)//2]
for path in to_remove:
# we could do them in a single go, but we can exceed the
# command line length -- lazy to optimize it
target.shell.run("rm -f %s" % path)
# Now copy all the files needed to boot to the root of the EFI
# system partition mounted in /boot; remember they are in /mnt/,
# our root partition
target.shell.run(
"time -p rsync --force --inplace /mnt/%(linux_kernel_file)s"
" /boot/%(linux_kernel_file_basename)s" % kws)
if kws.get("linux_initrd_file", None):
target.shell.run(
"time -p rsync --force --inplace /mnt/%(linux_initrd_file)s"
" /boot/%(linux_initrd_file_basename)s" % kws)
# we are the only one who cuts the cod here (yeah, direct Spanish
# translation for the reader's entertainment), and if not wipe'm to
# eternity; otherwise systemd will boot something prolly in
# alphabetical order, not what we want
target.shell.run("/usr/bin/rm -rf /boot/loader/entries/*.conf")
# remember paths to the bootloader are relative to /boot
# merge these two
tcf_boot_conf = """\
cat <<EOF > /boot/loader/entries/tcf-boot.conf
title TCF-driven local boot
linux /%(linux_kernel_file_basename)s
options %(linux_options)s
"""
if kws.get("linux_initrd_file", None):
tcf_boot_conf += "initrd /%(linux_initrd_file_basename)s\n"
tcf_boot_conf += "EOF\n"
target.shell.run(tcf_boot_conf % kws)
# Install new -- we wiped the /boot fs new anyway; if there are
# multiple options already, bootctl shall be able to handle it.
# Don't do variables in the any casewe will poke with them later
# on anyway. Why? Because there is space for a race condition that
# will leave us with the system booting off the localdisk vs the
# network for PXE--see efibootmgr_setup()
target.shell.run("bootctl update --no-variables"
" || bootctl install --no-variables;"
" sync")
# Now mess with the EFIbootmgr
# FIXME: make this a function and a configuration option (if the
# target does efibootmgr)
_efibootmgr_setup(target, boot_dev, 1)
# umount only if things go well
# Shall we try to unmount in case of error? nope, we are going to
# have to redo the whole thing anyway, so do not touch it, in case
# we are jumping in for manual debugging
target.shell.run("umount /dev/%(boot_part_dev)s" % kws)
def _complete(self, reporter, runid, hashid, tc_name, doc):
# Deliver to mongodb after adding a few more fields
doc['runid'] = runid
doc['hashid'] = hashid
doc['tc_name'] = tc_name
doc['timestamp'] = datetime.datetime.utcnow()
if runid:
doc['_id'] = runid + ":" + hashid
else:
doc['_id'] = hashid
doc['target_name'] = reporter.target_group.name \
if reporter.target_group else 'n/a'
if reporter.targets:
servers = set() # We don't care about reps in servers
target_types = [] # Here we want one per target
doc['targets'] = {}
# Note this is sorted by target-want-name, the names
# assigned by the testcase to the targets, so all the
# types and server lists are sorted by that.
for tgname in sorted(reporter.targets.keys()):
target = reporter.targets[tgname]
doc['targets'][tgname] = dict(server=target.rtb.aka,
id=target.id,
type=target.type,
bsp_model=target.bsp_model)
servers.add(target.rtb.aka)
if len(target.rt.get('bsp_models', {})) > 1:
target_types.append(target.type + ":" + target.bsp_model)
else:
target_types.append(target.type)
doc['target_types'] = ",".join(target_types)
doc['target_servers'] = ",".join(servers)
else:
doc['target_types'] = 'static'
doc['target_servers'] = 'static'
tags = {}
components = []
for tag in reporter._tags:
(value, _origin) = reporter.tag_get(tag, "", "")
tags[tag] = str(value)
if tag == 'components':
components = value.split()
doc['tags'] = tags
doc['components'] = components
for complete_hook in self.complete_hooks:
complete_hook(_tc, runid, hashid, tc_name, doc)
retry_count = 1
while retry_count <= 3:
if not self.results or self.made_in_pid != os.getpid():
self._mongo_setup()
try:
# We replace any existing reports for this _id -- if that
# is not to happen, provide a different runid...
self.results.find_one_and_replace({'_id': doc['_id']},
doc,
upsert=True)
break
except pymongo.errors.PyMongoError as e:
if retry_count <= 3:
raise tc.blocked_e(
"MongoDB error, can't record result: %s" % e)
else:
self.results = None
reporter.warning("MongoDB error, reconnecting (%d/3): %s" %
(e, retry_count))
def _linux_boot_guess_from_grub_cfg(target, _image):
"""
Extract kernel, initrd, kernel args from grub configuration
Parses a grub config file to extract which boot entries are
available, with their kernel and initrd files, the command line
options and the manu entry names.
Eliminate recovery and default, we shall be left with only one
that is the main boot option, the one that always has to be booted
that we will replicate.
A grub.cfg looks like a lot of configuration stuff, but at the
end, it boils down to entries like::
menuentry 'SLED 15-SP1' --class sled --class gnu-linux --class gnu --class os $menuentry_id_option 'gnulinux-simple-6bfcca25-0f4b-4d6c-87a0-c3bb25e5d512' {
load_video
set gfxpayload=keep
insmod gzio
insmod part_gpt
insmod btrfs
set root='hd0,gpt2'
if [ x$feature_platform_search_hint = xy ]; then
search --no-floppy --fs-uuid --set=root --hint-bios=hd0,gpt2 --hint-efi=hd0,gpt2 --hint-baremetal=ahci0,gpt2 --hint='hd0,gpt2' 6bfcca25-0f4b-4d6c-87a0-c3bb25e5d512
else
search --no-floppy --fs-uuid --set=root 6bfcca25-0f4b-4d6c-87a0-c3bb25e5d512
fi
echo 'Loading Linux 4.12.14-110-default ...'
linux /boot/vmlinuz-4.12.14-110-default root=UUID=6bfcca25-0f4b-4d6c-87a0-c3bb25e5d512 ${extra_cmdline} splash=silent resume=/dev/disk/by-id/ata-QEMU_HARDDISK_QM00001-part3 quiet crashkernel=173M,high
echo 'Loading initial ramdisk ...'
initrd /boot/initrd-4.12.14-110-default
}
There is also the older style::
default=0
timeout=6
splashimage=/grub/android-x86.xpm.gz
root (hd0,0)
title Android-x86 7.1-r2
kernel /android-7.1-r2/kernel quiet root=/dev/ram0 androidboot.selinux=permissive vmalloc=192M buildvariant=userdebug SRC=/android-7.1-r2
initrd /android-7.1-r2/initrd.img
title Android-x86 7.1-r2 (Debug mode)
kernel /android-7.1-r2/kernel root=/dev/ram0 androidboot.selinux=permissive vmalloc=192M buildvariant=userdebug DEBUG=2 SRC=/android-7.1-r2
initrd /android-7.1-r2/initrd.img
title Android-x86 7.1-r2 (Debug nomodeset)
kernel /android-7.1-r2/kernel nomodeset root=/dev/ram0 androidboot.selinux=permissive vmalloc=192M buildvariant=userdebug DEBUG=2 SRC=/android-7.1-r2
initrd /android-7.1-r2/initrd.img
title Android-x86 7.1-r2 (Debug video=LVDS-1:d)
kernel /android-7.1-r2/kernel video=LVDS-1:d root=/dev/ram0 androidboot.selinux=permissive vmalloc=192M buildvariant=userdebug DEBUG=2 SRC=/android-7.1-r2
initrd /android-7.1-r2/initrd.img
We ignore all but menuentry, linux, initrd; initrd might be missing
"""
# ignore errors if it does not exist
grub_cfg_path = target.shell.run(
# /mnt/grub|menu.lst is android ISO x86
# /mnt/boot|grub.cfg most other distros
# ignore failures if the find process errors out
"find /mnt/grub /mnt/boot"
# redirect erros to /dev/null so they don't pollute the output we need
" -iname grub.cfg -o -iname menu.lst 2> /dev/null"
" || true",
output=True,
trim=True)
grub_cfg_path = grub_cfg_path.strip()
if grub_cfg_path == "": # no grub.cfg to rely on
target.report_info("POS/uefi: found no grub config")
return None, None, None
target.report_info("POS/uefi: found grub config at %s" % grub_cfg_path)
# read grub in, but only the parts we care for--it's faster
grub_cfg = target.shell.run(
r" grep --color=never -w '\(menuentry\|title\|kernel\|linux\|initrd\)' %s"
% grub_cfg_path,
output=True,
trim=True)
# a grub entry
class _entry(object):
name = None
linux = None
linux_args = None
initrd = None
# we use a dictionary to remove duplicate entries
#
# we key the entries by the initrd, linux kernel and kernel args they
# use; if they are the same, it will be overriden and we won't have to
# pick one.
target._grub_entries = {}
entry = _entry()
def _entry_record():
entry_id = \
(entry.linux if entry.linux else "" ) \
+ (entry.initrd if entry.initrd else "") \
+ (entry.linux_args if entry.linux_args else "")
if entry_id != "": # record existing
target._grub_entries[entry_id] = entry
entry_count = 0
for line in grub_cfg.splitlines():
# match menuentry lines and save name, flushing previous data
# if any
m = _grub_menuentry_regex.search(line)
if m:
# new entry!, close the previous
_entry_record()
entry = _entry()
gd = m.groupdict()
# the entries will always exist because that's the regex
# search output, but we need to check they are not empty
if gd['name1']:
entry.name = gd['name1']
elif gd['name2']:
entry.name = gd['name2']
else:
entry.name = "entry #%d" % entry_count
entry_count += 1
continue
# match linux/initrd lines and extract values
m = _grub_linux_initrd_entry_regex.search(line)
if not m:
continue
gd = m.groupdict()
linux = gd.get('linux', None)
if linux:
entry.linux = linux
linux_args = gd.get('linux_args', None)
if linux:
# remove any ${SUBST} from the command line
linux_args = re.sub(_grub_var_regex, "", linux_args)
entry.linux_args = linux_args
initrd = gd.get('initrd', None)
if initrd:
entry.initrd = initrd
_entry_record() # record last entry
# delete recovery / rescue stuff, we don't use it
for entry_id in list(target._grub_entries.keys()):
entry = target._grub_entries[entry_id]
if _ignore_boot_entry(target, entry.name,
grub_cfg_path.replace("/mnt", "")):
del target._grub_entries[entry_id]
if len(target._grub_entries) > 1:
entries = pprint.pformat(
[i.__dict__ for i in target._grub_entries.values()])
raise tc.blocked_e(
"more than one Linux kernel entry; I don't know "
"which one to use", dict(target=target, entries=entries))
if not target._grub_entries: # can't find?
del target._grub_entries # need no more
return None, None, None
entry = target._grub_entries.values()[0]
del target._grub_entries # need no more
# note we assume the grub.cfg entries are in [/mnt]/boot because
# grub.cfg is in /boot and there is usually a filesystem just for
# /boot, thus the entries are relative.
if entry.linux:
entry.linux = "/boot/" + entry.linux
if initrd:
entry.initrd = "/boot/" + entry.initrd
target.report_info(
"POS/EFI: %s: scanning found kernel %s initrd %s args %s" %
(grub_cfg_path, entry.linux, entry.initrd, entry.linux_args))
return entry.linux, entry.initrd, entry.linux_args
def _linux_boot_guess_from_lecs(target, _image):
"""
Setup a Linux kernel to boot using Gumniboot
"""
# ignore errors if it does not exist
lecs = target.shell.run(
r"find /mnt/boot/loader/entries -type f -iname \*.conf || true",
output = True)
# this returns something like
#
# find /mnt/boot/loader/entries -type f -iname \*.conf
# /mnt/boot/loader/entries/Clear-linux-native-4.18.13-644.conf
# /mnt/boot/loader/entries/Something-else.conf
# 10 $
#
# Filter just the output we care for
lecl = []
for lec in lecs.split("\n"):
lec = lec.strip()
if not lec.startswith("/mnt/boot/loader/entries/"):
continue
if _ignore_boot_entry(target, os.path.basename(lec),
os.path.dirname(lec)):
continue
lecl.append(lec)
target.report_info("Loader Entry found: %s" % lec, dlevel = 1)
if len(lecl) > 1:
raise tc.blocked_e(
"multiple loader entries in /boot, do not "
"know which one to use: " + " ".join(lecl),
dict(target = target))
elif len(lecl) == 0:
return None, None, None
# fallthrough, only one entry
lec = lecl[0]
output = target.shell.run('cat %s' % lec, output = True)
kernel = None
initrd = None
options = None
# read a loader entry, extract the kernel, initramfs and options
# thanks Loader Entry Specification for making them single liners...
dibs_regex = re.compile(r"^\s*(?P<command>linux|initrd|efi|options)\s+"
"(?P<value>[^\n]+)\n?")
for line in output.splitlines():
m = dibs_regex.match(line)
if not m:
continue
d = m.groupdict()
command = d['command']
value = d['value']
if command == 'linux':
kernel = value
elif command == 'efi':
kernel = value
elif command == 'initrd':
initrd = value
elif command == 'options':
options = value
# note we assume the LEC entries are in [/mnt]/boot because LEC
# specifies them relateive to the filesystem
if kernel:
kernel = "/boot/" + kernel
if initrd:
initrd = "/boot/" + initrd
return kernel, initrd, options
def _disk_partition(target):
# we assume we are going to work on the boot device
device_basename = target.kws['pos_boot_dev']
device = "/dev/" + device_basename
target.shell.run('swapoff -a || true') # in case we autoswapped
# find device size (FIXME: Linux specific)
dev_info = None
for blockdevice in target.pos.fsinfo.get('blockdevices', []):
if blockdevice['name'] == device_basename:
dev_info = blockdevice
break
else:
raise tc.error_e(
"%s: can't find information about this block device -- is "
"the right pos_boot_device set in the configuration?" %
device_basename, dict(fsinfo=pprint.pformat(target.pos.fsinfo)))
size_gb = int(dev_info['size']) / 1024 / 1024 / 1024
target.report_info("POS: %s is %d GiB in size" % (device, size_gb),
dlevel=2)
partsizes = target.kws.get('pos_partsizes', None)
if partsizes == None:
raise tc.blocked_e(
"Can't partition target, it doesn't "
"specify pos_partsizes tag", {'target': target})
partsize_l = partsizes.split(":")
partsize_l = [int(_partsize) for _partsize in partsize_l]
boot_size = partsize_l[0]
swap_size = partsize_l[1]
scratch_size = partsize_l[2]
root_size = partsize_l[3]
# note we set partition #0 as boot
cmdline = """parted -a optimal -ms %(device)s unit GiB \
mklabel gpt \
mkpart primary fat32 0%% %(boot_size)s \
set 1 boot on \
mkpart primary linux-swap %(boot_size)s %(swap_end)s \
mkpart primary ext4 %(swap_end)s %(scratch_end)s \
""" % dict(
device=device,
boot_size=boot_size,
swap_end=boot_size + swap_size,
scratch_end=boot_size + swap_size + scratch_size,
)
offset = boot_size + swap_size + scratch_size
root_devs = [] # collect the root devices
pid = 4
while offset + root_size < size_gb:
cmdline += ' mkpart primary ext4 %d %d' % (offset, offset + root_size)
offset += root_size
root_devs.append(device_basename + target.kws['p_prefix'] + "%d" % pid)
pid += 1
target.shell.run(cmdline)
# Now set the root device information, so we can pick stuff to
# format quick
for root_dev in root_devs:
target.property_set('pos_root_' + root_dev, "EMPTY")
# Re-read partition tables
target.shell.run('partprobe %s' % device)
# now format filesystems
#
# note we only format the system boot partition (1), the linux
# swap(2) and the linux scratch space (3)
boot_dev = device + target.kws['p_prefix'] + "1"
swap_dev = device + target.kws['p_prefix'] + "2"
home_dev = device + target.kws['p_prefix'] + "3"
# Note: use FAT vs VFAT: vfat name translation creates issues when
# doing long file names; fat32 does not have that problem.
target.shell.run("mkfs.fat -F32 -n TCF-BOOT " + boot_dev)
target.shell.run("mkswap -L tcf-swap " + swap_dev)
target.shell.run("mkfs.ext4 -FqL tcf-scratch " + home_dev)
def _linux_boot_guess_from_grub_cfg(target, _image):
"""
Extract kernel, initrd, kernel args from grub configuration
Parses a grub config file to extract which boot entries are
available, with their kernel and initrd files, the command line
options and the manu entry names.
Eliminate recovery and default, we shall be left with only one
that is the main boot option, the one that always has to be booted
that we will replicate.
A grub.cfg looks like a lot of configuration stuff, but at the
end, it boils down to entries like::
menuentry 'SLED 15-SP1' --class sled --class gnu-linux --class gnu --class os $menuentry_id_option 'gnulinux-simple-6bfcca25-0f4b-4d6c-87a0-c3bb25e5d512' {
load_video
set gfxpayload=keep
insmod gzio
insmod part_gpt
insmod btrfs
set root='hd0,gpt2'
if [ x$feature_platform_search_hint = xy ]; then
search --no-floppy --fs-uuid --set=root --hint-bios=hd0,gpt2 --hint-efi=hd0,gpt2 --hint-baremetal=ahci0,gpt2 --hint='hd0,gpt2' 6bfcca25-0f4b-4d6c-87a0-c3bb25e5d512
else
search --no-floppy --fs-uuid --set=root 6bfcca25-0f4b-4d6c-87a0-c3bb25e5d512
fi
echo 'Loading Linux 4.12.14-110-default ...'
linux /boot/vmlinuz-4.12.14-110-default root=UUID=6bfcca25-0f4b-4d6c-87a0-c3bb25e5d512 ${extra_cmdline} splash=silent resume=/dev/disk/by-id/ata-QEMU_HARDDISK_QM00001-part3 quiet crashkernel=173M,high
echo 'Loading initial ramdisk ...'
initrd /boot/initrd-4.12.14-110-default
}
There is also the older style::
default=0
timeout=6
splashimage=/grub/android-x86.xpm.gz
root (hd0,0)
title Android-x86 7.1-r2
kernel /android-7.1-r2/kernel quiet root=/dev/ram0 androidboot.selinux=permissive vmalloc=192M buildvariant=userdebug SRC=/android-7.1-r2
initrd /android-7.1-r2/initrd.img
title Android-x86 7.1-r2 (Debug mode)
kernel /android-7.1-r2/kernel root=/dev/ram0 androidboot.selinux=permissive vmalloc=192M buildvariant=userdebug DEBUG=2 SRC=/android-7.1-r2
initrd /android-7.1-r2/initrd.img
title Android-x86 7.1-r2 (Debug nomodeset)
kernel /android-7.1-r2/kernel nomodeset root=/dev/ram0 androidboot.selinux=permissive vmalloc=192M buildvariant=userdebug DEBUG=2 SRC=/android-7.1-r2
initrd /android-7.1-r2/initrd.img
title Android-x86 7.1-r2 (Debug video=LVDS-1:d)
kernel /android-7.1-r2/kernel video=LVDS-1:d root=/dev/ram0 androidboot.selinux=permissive vmalloc=192M buildvariant=userdebug DEBUG=2 SRC=/android-7.1-r2
initrd /android-7.1-r2/initrd.img
We ignore all but menuentry, linux, initrd; initrd might be missing
"""
# ignore errors if it does not exist
grub_cfg_path = target.shell.run(
# /mnt/grub|menu.lst is android ISO x86
# /mnt/boot|grub.cfg most other distros
# ignore failures if the find process errors out
"find /mnt/grub /mnt/boot"
# redirect erros to /dev/null so they don't pollute the output we need
" -iname grub.cfg -o -iname menu.lst 2> /dev/null"
" || true",
output = True, trim = True)
grub_cfg_path = grub_cfg_path.strip()
if grub_cfg_path == "": # no grub.cfg to rely on
target.report_info("POS/uefi: found no grub config")
return None, None, None
target.report_info("POS/uefi: found grub config at %s" % grub_cfg_path)
# read grub in, but only the parts we care for--it's faster
grub_cfg = target.shell.run(
r" grep --color=never -w '\(menuentry\|title\|kernel\|linux\|initrd\)' %s"
% grub_cfg_path,
output = True, trim = True)
# a grub entry
class _entry(object):
name = None
linux = None
linux_args = None
initrd = None
# we use a dictionary to remove duplicate entries
#
# we key the entries by the initrd, linux kernel and kernel args they
# use; if they are the same, it will be overriden and we won't have to
# pick one.
target._grub_entries = {}
entry = _entry()
def _entry_record():
entry_id = \
(entry.linux if entry.linux else "" ) \
+ (entry.initrd if entry.initrd else "") \
+ (entry.linux_args if entry.linux_args else "")
if entry_id != "": # record existing
target._grub_entries[entry_id] = entry
entry_count = 0
for line in grub_cfg.splitlines():
# match menuentry lines and save name, flushing previous data
# if any
m = _grub_menuentry_regex.search(line)
if m:
# new entry!, close the previous
_entry_record()
entry = _entry()
gd = m.groupdict()
# the entries will always exist because that's the regex
# search output, but we need to check they are not empty
if gd['name1']:
entry.name = gd['name1']
elif gd['name2']:
entry.name = gd['name2']
else:
entry.name = "entry #%d" % entry_count
entry_count += 1
continue
# match linux/initrd lines and extract values
m = _grub_linux_initrd_entry_regex.search(line)
if not m:
continue
gd = m.groupdict()
linux = gd.get('linux', None)
if linux:
entry.linux = linux
linux_args = gd.get('linux_args', None)
if linux:
# remove any ${SUBST} from the command line
linux_args = re.sub(_grub_var_regex, "", linux_args)
entry.linux_args = linux_args
initrd = gd.get('initrd', None)
if initrd:
entry.initrd = initrd
_entry_record() # record last entry
# delete recovery / rescue stuff, we don't use it
for entry_id in list(target._grub_entries.keys()):
entry = target._grub_entries[entry_id]
if _ignore_boot_entry(target, entry.name,
grub_cfg_path.replace("/mnt", "")):
del target._grub_entries[entry_id]
if len(target._grub_entries) > 1:
entries = pprint.pformat([ i.__dict__
for i in target._grub_entries.values() ])
raise tc.blocked_e(
"more than one Linux kernel entry; I don't know "
"which one to use",
dict(target = target, entries = entries))
if not target._grub_entries: # can't find?
del target._grub_entries # need no more
return None, None, None
entry = target._grub_entries.values()[0]
del target._grub_entries # need no more
return entry.linux, entry.initrd, entry.linux_args
def mount_fs(target, image, boot_dev):
"""
Boots a root filesystem on /mnt
The partition used as a root filesystem is picked up based on the
image that is going to be installed; we look for one that has the
most similar image already installed and pick that.
:returns: name of the root partition device
"""
# does the disk have a partition scheme we recognize?
pos_partsizes = target.rt['pos_partsizes']
# the name we'll give to the boot partition; see
# _disk_partition(); if we can't find it, we assume the things
# needs to be repartitioned. Note it includes the sizes, so if we
# change the sizes in the config it reformats automatically. #
# TCF-multiroot-NN:NN:NN:NN
target._boot_label_name = "TCF-multiroot-" + pos_partsizes
pos_reinitialize_force = True
boot_dev_base = os.path.basename(boot_dev)
child = target.pos.fsinfo_get_child_by_partlabel(boot_dev_base,
target._boot_label_name)
if child:
pos_reinitialize_force = False
else:
target.report_info("POS: repartitioning due to different"
" partition schema")
pos_reinitialize = target.property_get("pos_reinitialize", False)
if pos_reinitialize:
target.report_info("POS: repartitioning per pos_reinitialize "
"property")
if pos_reinitialize or pos_reinitialize_force:
# Need to reinit the partition table (we were told to by
# setting pos_repartition to anything or we didn't recognize
# the existing partitioning scheme)
for tag in target.rt.keys():
# remove pos_root_*, as they don't apply anymore
if tag.startswith("pos_root_"):
target.property_set(tag, None)
_disk_partition(target)
target.pos.fsinfo_read(target._boot_label_name)
target.property_set('pos_reinitialize', None)
root_part_dev = _rootfs_guess(target, image, boot_dev)
# save for other functions called later
target.root_part_dev = root_part_dev
root_part_dev_base = os.path.basename(root_part_dev)
image_prev = target.property_get("pos_root_" + root_part_dev_base,
"nothing")
target.report_info("POS: will use %s for root partition (had %s before)" %
(root_part_dev, image_prev))
# fsinfo looks like described in target.pos.fsinfo_read()
dev_info = None
for blockdevice in target.pos.fsinfo.get('blockdevices', []):
for child in blockdevice.get('children', []):
if child['name'] == root_part_dev_base:
dev_info = child
if dev_info == None:
# it cannot be we might have to repartition because at this
# point *we* have partitioned.
raise tc.error_e(
"Can't find information for root device %s in FSinfo array" %
root_part_dev_base, dict(fsinfo=target.pos.fsinfo))
# what format does it currently have?
current_fstype = dev_info.get('fstype', 'ext4')
# What format does it have to have?
#
# Ok, here we need to note that we can't have multiple root
# filesystems with the same UUID or LABEL, so the image can't rely
# on UUIDs
#
img_fss = target.pos.metadata.get('filesystems', {})
if '/' in img_fss:
# a common origin is ok because the YAML schema forces both
# fstype and mkfs_opts to be specified
origin = "image's /.tcf.metadata.yaml"
fsdata = img_fss.get('/', {})
else:
origin = "defaults @" + commonl.origin_get(0)
fsdata = {}
fstype = fsdata.get('fstype', 'ext4')
mkfs_opts = fsdata.get('mkfs_opts', '-Fj')
# do they match?
if fstype != current_fstype:
target.report_info(
"POS: reformatting %s because current format is '%s' and "
"'%s' is needed (per %s)" %
(root_part_dev, current_fstype, fstype, origin))
_mkfs(target, root_part_dev, fstype, mkfs_opts)
else:
target.report_info(
"POS: no need to reformat %s because current format is '%s' and "
"'%s' is needed (per %s)" %
(root_part_dev, current_fstype, fstype, origin),
dlevel=1)
for try_count in range(3):
target.report_info("POS: mounting root partition %s onto /mnt "
"to image [%d/3]" % (root_part_dev, try_count))
# don't let it fail or it will raise an exception, so we
# print FAILED in that case to look for stuff; note the
# double apostrophe trick so the regex finder doens't trip
# on the command
output = target.shell.run("mount %s /mnt || echo FAI''LED" %
root_part_dev,
output=True)
# What did we get?
if 'FAILED' in output:
if 'special device ' + root_part_dev \
+ ' does not exist.' in output:
_disk_partition(target)
target.pos.fsinfo_read(target._boot_label_name)
else:
# ok, this probably means probably the partitions are not
# formatted; so let's just reformat and retry
_mkfs(target, root_part_dev, fstype, mkfs_opts)
else:
target.report_info("POS: mounted %s onto /mnt to image" %
root_part_dev)
return root_part_dev # it worked, we are done
# fall through, retry
else:
raise tc.blocked_e("POS: Tried to mount too many times and failed",
dict(target=target))
def flash_spec_parse(self, flash_image_s=None):
"""Parse a images to flash specification in a string (that might be
taken from the environment
The string describing what to flash is in the form::
[[no-]soft] [[no-]upload] IMAGE:NAME[ IMAGE:NAME[..]]]
- *soft*: flash in soft mode (default *False) (see
:meth:`target.images.flash
<tcfl.target_ext_images.extension.flash>`) the image(s) will
only be flashed if the image to be flashed is different than the
last image that was flashed.
- *upload*: flash in soft mode (default *True*) (see
:meth:`target.images.flash
<tcfl.target_ext_images.extension.flash>`). The file will be
uploaded to the server first or it will be assumed it is already
present in the server.
- *IMAGETYPE:FILENAME* flash file *FILENAME* in flash destination
*IMAGETYPE*; *IMAGETYPE* being any of the image
destinations the target can flash; can be found with::
$ tcf image-ls TARGETNAME
or from the inventory::
$ tcf get TARGETNAME -p interfaces.images
The string specification will be taken, in this order from the
following list
- the *image_flash* parameter
- environment *IMAGE_FLASH_<TYPE>*
- environment *IMAGE_FLASH_<FULLID>*
- environment *IMAGE_FLASH_<ID>*
- environment *IMAGE_FLASH*
With *TYPE*, *FULLID* and *ID* sanitized to any character outside
of the ranges *[a-zA-Z0-9_]* replaced with an underscore (*_*).
**Example**
::
$ export IMAGE_FLASH_TYPE1="soft bios:path/to/bios.bin"
$ tcf run test_SOMESCRIPT.py
if *test_SOMESCRIPT.py* uses this template, every invocation of it
on a machine of type TYPE1 will result on the file
*path/to/bios.bin* being flashed on the *bios* location (however,
because of *soft*, if it was already flashed before, it will be
skipped).
:return: a tuple of
>>> ( DICT, UPLOAD, SOFT )
- *DICT*: Dictionary is a dictionary of IMAGETYPE/file name to flash:
>>> {
>>> IMGTYPE1: IMGFILE1,
>>> IMGTYPE2: IMGFILE2,
>>> ...
>>> }
- *UPLOAD*: boolean indicating if the user wants the files to be
uploaded (*True*, default) or to assume they are already in
the server (*False*).
- *SOFT*: flash in soft mode (*True*) or not (*False*, default
if not given).
"""
target = self.target
if not flash_image_s:
# empty, load from environment
target_id_safe = commonl.name_make_safe(
target.id, string.ascii_letters + string.digits)
target_fullid_safe = commonl.name_make_safe(
target.fullid, string.ascii_letters + string.digits)
target_type_safe = commonl.name_make_safe(
target.type, string.ascii_letters + string.digits)
source = None # keep pylint happy
sourcel = [
# go from most specifcy to most generic
"IMAGE_FLASH_%s" % target_fullid_safe,
"IMAGE_FLASH_%s" % target_id_safe,
"IMAGE_FLASH_%s" % target_type_safe,
"IMAGE_FLASH",
]
for source in sourcel:
flash_image_s = os.environ.get(source, None)
if flash_image_s != None:
break
else:
target.report_info(
"skipping image flashing (no function argument nor environment: %s)"
% " ".join(sourcel))
return {}, False, False
else:
source = "function argument"
# verify the format
if not self._image_flash_regex.search(flash_image_s):
raise tc.blocked_e(
"image specification in %s does not conform to the form"
" [[no-]soft] [[no-]upload] IMAGE:NAME[ IMAGE:NAME[..]]]" %
source, dict(target=target))
image_flash = {}
soft = False
upload = True
for entry in flash_image_s.split(" "):
if not entry: # empty spaces...welp
continue
if entry == "soft":
soft = True
continue
if entry == "no-soft":
soft = False
continue
if entry == "upload":
upload = True
continue
if entry == "no-upload":
upload = False
continue
name, value = entry.split(":", 1)
image_flash[name] = value
return image_flash, upload, soft
def console_rx_eval(expecter,
target,
regex,
console=None,
_timeout=None,
result=None,
uid=None):
"""
Check what came on a console and act on it
:param str uid: (optional) identifier to use to store offset data
"""
if hasattr(regex, "pattern"):
what = regex.pattern
else:
what = regex
regex = re.compile(re.escape(regex))
if not uid:
uid = console_mk_uid(target, what, console, _timeout, result)
console_id_name, console_code = console_mk_code(target, console)
# These were set by the poller
of = expecter.buffers.get(console_code, None)
if of == None:
# FIXME: debug lof output here? expecter->tc backlink?
return None
ofd = of.fileno()
ts = time.time()
# Get the offset we saved before as the last part where we looked
# at. If none, then get the last offset the poller has
# recorded. Otherwise, just default to look from the start
# Note the idea is each eval function has a different offset where
# it is looking at. Likewise the poller for each console.
offset_poller_code = "offset_" + console_code
offset = expecter.buffers_persistent.get(
uid, expecter.buffers_persistent.get(offset_poller_code, 0))
if _timeout != False:
timeout = expecter.timeout if _timeout == None else _timeout
# We can do timeout checks that provide better information
# than a generic 'timeout'
if ts - expecter.ts0 > timeout:
of.seek(offset) # so we report console from where searched
raise tc.error_e(
"expected console output '%s' from console '%s:%s' " \
"NOT FOUND after %.1f s" \
% (what, target.id, console_id_name, ts - expecter.ts0),
{ 'target': target, "console output": of })
# mmap the whole file (which doesn't alter the file pointer)
#
# We have to mmap as the file might be getting huge and thus,
# reading line by line might be dumb.
#
# However, we only search starting at @offset, which is set later
# to the last success searching we had. So we shan't really map
# the whole file, shall map on demand.
stat_info = os.fstat(ofd)
if stat_info.st_size == 0: # Nothing to read
return None
with contextlib.closing(
mmap.mmap(of.fileno(), 0, mmap.MAP_PRIVATE, mmap.PROT_READ,
0)) as mapping:
target.report_info("looking for `%s` in console %s:%s @%d-%d at "
"%.2fs [%s]" %
(what, target.fullid, console_id_name, offset,
stat_info.st_size, ts - expecter.ts0, of.name),
dlevel=3)
m = regex.search(mapping[offset:])
if m:
new_offset = offset + m.end()
expecter.buffers_persistent[uid] = new_offset
if result == None or result == "pass":
# raising pass gets stopped at expecter.run(), so we
# print instead, so we can see the console
offset_tip = of.tell() # we report console from where searched
of.seek(offset) # so we report console from where searched
target.report_pass(
"found expected `%s` in console `%s:%s` at %.2fs @%d" %
(what, target.fullid, console_id_name, ts - expecter.ts0,
offset + m.start()), {"console output": of},
dlevel=1,
alevel=2)
of.seek(offset_tip)
raise tc.pass_e(
"found expected `%s` in console `%s:%s` at %.2fs" %
(what, target.fullid, console_id_name, ts - expecter.ts0),
{'target': target})
elif result == "fail":
of.seek(offset) # so we report console from where searched
raise tc.failed_e(
"found expected (for failure) `%s` in console "
"`%s:%s` at %.2fs" %
(what, target.fullid, console_id_name, ts - expecter.ts0),
{
'target': target,
"console output": of
})
elif result == "error" or result == "errr":
of.seek(offset) # so we report console from where searched
raise tc.error_e(
"found expected (for error) `%s` in console "
"`%s:%s` at %.2fs" %
(what, target.fullid, console_id_name, ts - expecter.ts0),
{
'target': target,
"console output": of
})
elif result == "skip":
of.seek(offset) # so we report console from where searched
raise tc.skip_e(
"found expected (for skip) `%s` in console "
"'%s:%s' at %.2fs" %
(what, target.fullid, console_id_name, ts - expecter.ts0),
{
'target': target,
"console output": of
})
else:
of.seek(offset) # so we report console from where searched
raise tc.blocked_e("BUG: invalid result requested (%s)" %
result)
return None
def _linux_boot_guess_from_lecs(target, _image):
"""
Setup a Linux kernel to boot using Gumniboot
"""
# ignore errors if it does not exist
lecs = target.shell.run(
r"find /mnt/boot/loader/entries -type f -iname \*.conf || true",
output=True)
# this returns something like
#
# find /mnt/boot/loader/entries -type f -iname \*.conf
# /mnt/boot/loader/entries/Clear-linux-native-4.18.13-644.conf
# /mnt/boot/loader/entries/Something-else.conf
# 10 $
#
# Filter just the output we care for
lecl = []
for lec in lecs.split("\n"):
lec = lec.strip()
if not lec.startswith("/mnt/boot/loader/entries/"):
continue
if _ignore_boot_entry(target, os.path.basename(lec),
os.path.dirname(lec)):
continue
lecl.append(lec)
target.report_info("Loader Entry found: %s" % lec, dlevel=1)
if len(lecl) > 1:
raise tc.blocked_e(
"multiple loader entries in /boot, do not "
"know which one to use: " + " ".join(lecl), dict(target=target))
elif len(lecl) == 0:
return None, None, None
# fallthrough, only one entry
lec = lecl[0]
output = target.shell.run('cat %s' % lec, output=True)
kernel = None
initrd = None
options = None
# read a loader entry, extract the kernel, initramfs and options
# thanks Loader Entry Specification for making them single liners...
dibs_regex = re.compile(r"^\s*(?P<command>linux|initrd|efi|options)\s+"
"(?P<value>[^\n]+)\n?")
for line in output.splitlines():
m = dibs_regex.match(line)
if not m:
continue
d = m.groupdict()
command = d['command']
value = d['value']
if command == 'linux':
kernel = value
elif command == 'efi':
kernel = value
elif command == 'initrd':
initrd = value
elif command == 'options':
options = value
# note we assume the LEC entries are in [/mnt]/boot because LEC
# specifies them relateive to the filesystem
if kernel:
kernel = "/boot/" + kernel
if initrd:
initrd = "/boot/" + initrd
return kernel, initrd, options
def boot_config_multiroot(target, boot_dev, image):
"""
Configure the target to boot using the multiroot
"""
boot_dev = target.kws['pos_boot_dev']
# were we have mounted the root partition
root_dir = "/mnt"
linux_kernel_file, linux_initrd_file, linux_options = \
_linux_boot_guess(target, image)
if linux_kernel_file == None:
raise tc.blocked_e("Cannot guess a Linux kernel to boot",
dict(target=target))
# remove absolutization (some specs have it), as we need to copy from
# mounted filesystems
if os.path.isabs(linux_kernel_file):
linux_kernel_file = linux_kernel_file[1:]
if linux_initrd_file and os.path.isabs(linux_initrd_file):
linux_initrd_file = linux_initrd_file[1:]
if linux_options == None or linux_options == "":
target.report_info("WARNING! can't figure out Linux cmdline "
"options, taking defaults")
# below we'll add more stuff
linux_options = "console=tty0 root=SOMEWHERE"
# MULTIROOT: indicate which image has been flashed to this
# partition
# saved by pos_multiroot.mountfs
root_part_dev = target.root_part_dev
root_part_dev_base = os.path.basename(root_part_dev)
target.property_set('pos_root_' + root_part_dev_base, image)
# /boot EFI system partition is always /dev/DEVNAME1 (first
# partition), we partition like that
# FIXME: we shouldn't harcode this
boot_part_dev = boot_dev + target.kws['p_prefix'] + "1"
kws = dict(
boot_dev=boot_dev,
boot_part_dev=boot_part_dev,
root_part_dev=root_part_dev,
root_part_dev_base=root_part_dev_base,
root_dir=root_dir,
linux_kernel_file=linux_kernel_file,
linux_kernel_file_basename=os.path.basename(linux_kernel_file),
linux_initrd_file=linux_initrd_file,
linux_options=linux_options,
)
if linux_initrd_file:
kws['linux_initrd_file_basename'] = os.path.basename(linux_initrd_file)
else:
kws['linux_initrd_file_basename'] = None
kws.update(target.kws)
if linux_options:
#
# Maybe mess with the Linux boot options
#
target.report_info("linux cmdline options: %s" % linux_options)
# FIXME: can this come from config?
linux_options_replace = {
# we want to use hard device name rather than LABELS/UUIDs, as
# we have reformated and those will have changed
"root": "/dev/%(root_part_dev_base)s" % kws,
# we have created this in pos_multiroot and will only
# replace it if the command line option is present.
"resume": "/dev/disk/by-label/tcf-swap",
}
# FIXME: can this come from config?
# We harcode a serial console on the device where we know the
# framework is listening
linux_options_append = \
"console=%(linux_serial_console_default)s,115200n8" % kws
linux_options_append += " " + target.rt.get('linux_options_append', "")
linux_options += " " + linux_options_append
for option, value in linux_options_replace.iteritems():
regex = re.compile(r"\b" + option + r"=\S+")
if regex.search(linux_options):
linux_options = re.sub(
regex, option + "=" + linux_options_replace[option],
linux_options)
else:
linux_options += " " + option + "=" + value
kws['linux_options'] = linux_options
target.report_info("linux cmdline options (modified): %s" %
linux_options)
# Now generate the UEFI system partition that will boot the
# system; we always override it, so we don't need to decide if it
# is corrupted or whatever; we'll mount it in /boot (which now is
# the POS /boot)
#
# Mount the /boot fs
#
# Try to assume it is ok, try to repair it if not; rsync the
# kernels in there, it is faster for repeated operation/
#
target.report_info("POS/EFI: checking %(boot_part_dev)s" % kws)
output = target.shell.run("fsck.fat -aw /dev/%(boot_part_dev)s || true" %
kws,
output=True,
trim=True)
# FIXME: parse the output to tell if there was a problem; when bad
# but recovered, we'll see
#
# 0x41: Dirty bit is set. Fs was not properly unmounted and some data may be corrupt.
# Automatically removing dirty bit.
# Performing changes.
# /dev/sda1: 11 files, 4173/261372 clusters
# When ok
#
## $ sudo fsck.vfat -wa /dev/nvme0n1p1
## fsck.fat 4.1 (2017-01-24)
## /dev/sda1: 39 files, 2271/33259 clusters
# When really hosed it won't print the device line, so we look for
# that
#
## $ fsck.vfat -wa /dev/trash
## fsck.fat 4.1 (2017-01-24)
## Logical sector size (49294 bytes) is not a multiple of the physical sector size.
good_regex = re.compile(
"^/dev/%(boot_part_dev)s: [0-9]+ files, "
"[0-9]+/[0-9]+ clusters$" % kws, re.MULTILINE)
if not good_regex.search(output):
target.report_info(
"POS/EFI: /dev/%(boot_part_dev)s: formatting EFI "
"filesystem, fsck couldn't fix it" % kws, dict(output=output))
target.shell.run("mkfs.fat -F32 /dev/%(boot_part_dev)s; sync" % kws)
target.report_info("POS/EFI: /dev/%(boot_part_dev)s: mounting in /boot" %
kws)
target.shell.run(" mount /dev/%(boot_part_dev)s /boot"
" && mkdir -p /boot/loader/entries " % kws)
# Do we have enough space? if not, remove the oldest stuff that is
# not the file we are looking for
# This prints
## $ df --output=pcent /boot
## Use%
## 6%
output = target.shell.run("df --output=pcent /boot", output=True)
regex = re.compile(r"^\s*(?P<percent>[\.0-9]+)%$", re.MULTILINE)
match = regex.search(output)
if not match:
raise tc.error_e("Can't determine the amount of free space in /boot",
dict(output=output))
used_space = float(match.groupdict()['percent'])
if used_space > 75:
target.report_info(
"POS/EFI: /dev/%(boot_part_dev)s: freeing up space" % kws)
# List files in /boot, sort by last update (when we rsynced
# them)
## 2018-10-29+08:48:48.0000000000 84590 /boot/EFI/BOOT/BOOTX64.EFI
## 2018-10-29+08:48:48.0000000000 84590 /boot/EFI/systemd/systemd-bootx64.efi
## 2019-05-14+13:25:06.0000000000 7192832 /boot/vmlinuz-4.12.14-110-default
## 2019-05-14+13:25:08.0000000000 9688340 /boot/initrd-4.12.14-110-default
## 2019-05-14+13:25:14.0000000000 224 /boot/loader/entries/tcf-boot.conf
## 2019-05-14+13:25:14.0000000000 54 /boot/loader/loader.conf
output = target.shell.run(
# that double \\ needed so the shell is the one using it
# as a \t, not python converting to a sequence
"find /boot/ -type f -printf '%T+\\t%s\\t%p\\n' | sort",
output=True,
trim=True)
# delete the first half entries over 300k except those that
# match the kernels we are installing
to_remove = []
_linux_initrd_file = kws.get("linux_initrd_file", "%%NONE%%")
if linux_initrd_file == None:
_linux_initrd_file = "%%NONE%%"
for line in output.splitlines():
_timestamp, size_s, path = line.split(None, 3)
size = int(size_s)
if size > 300 * 1024 \
and not kws['linux_kernel_file_basename'] in path \
and not _linux_initrd_file in path:
to_remove.append(path)
# get older half and wipe it--this means the first half, as we
# sort from older to newer
to_remove = to_remove[:len(to_remove) // 2]
for path in to_remove:
# we could do them in a single go, but we can exceed the
# command line length -- lazy to optimize it
target.shell.run("rm -f %s" % path)
# Now copy all the files needed to boot to the root of the EFI
# system partition mounted in /boot; remember they are in /mnt/,
# our root partition
target.shell.run(
"time -p rsync --force --inplace /mnt/%(linux_kernel_file)s"
" /boot/%(linux_kernel_file_basename)s" % kws)
if kws.get("linux_initrd_file", None):
target.shell.run(
"time -p rsync --force --inplace /mnt/%(linux_initrd_file)s"
" /boot/%(linux_initrd_file_basename)s" % kws)
# we are the only one who cuts the cod here (yeah, direct Spanish
# translation for the reader's entertainment), and if not wipe'm to
# eternity; otherwise systemd will boot something prolly in
# alphabetical order, not what we want
target.shell.run("/usr/bin/rm -rf /boot/loader/entries/*.conf")
# remember paths to the bootloader are relative to /boot
# merge these two
tcf_boot_conf = """\
cat <<EOF > /boot/loader/entries/tcf-boot.conf
title TCF-driven local boot
linux /%(linux_kernel_file_basename)s
options %(linux_options)s
"""
if kws.get("linux_initrd_file", None):
tcf_boot_conf += "initrd /%(linux_initrd_file_basename)s\n"
tcf_boot_conf += "EOF\n"
target.shell.run(tcf_boot_conf % kws)
# Install new -- we wiped the /boot fs new anyway; if there are
# multiple options already, bootctl shall be able to handle it.
# Don't do variables in the any casewe will poke with them later
# on anyway. Why? Because there is space for a race condition that
# will leave us with the system booting off the localdisk vs the
# network for PXE--see efibootmgr_setup()
target.shell.run("bootctl update --no-variables"
" || bootctl install --no-variables;"
" sync")
# Now mess with the EFIbootmgr
# FIXME: make this a function and a configuration option (if the
# target does efibootmgr)
_efibootmgr_setup(target, boot_dev, 1)
# umount only if things go well
# Shall we try to unmount in case of error? nope, we are going to
# have to redo the whole thing anyway, so do not touch it, in case
# we are jumping in for manual debugging
target.shell.run("umount /dev/%(boot_part_dev)s" % kws)
def _disk_partition(target):
# we assume we are going to work on the boot device
device_basename = target.kws['pos_boot_dev']
device = "/dev/" + device_basename
target.shell.run('swapoff -a || true') # in case we autoswapped
# find device size (FIXME: Linux specific)
dev_info = target.pos.fsinfo_get_block(device_basename)
size_gb = int(dev_info['size']) / 1024 / 1024 / 1024
target.report_info("POS: %s is %d GiB in size" % (device, size_gb),
dlevel=2)
partsizes = target.kws.get('pos_partsizes', None)
if partsizes == None:
raise tc.blocked_e(
"Can't partition target, it doesn't "
"specify pos_partsizes tag", {'target': target})
partsize_l = partsizes.split(":")
partsize_l = [int(_partsize) for _partsize in partsize_l]
boot_size = partsize_l[0]
swap_size = partsize_l[1]
scratch_size = partsize_l[2]
root_size = partsize_l[3]
# note we set partition #0 as boot
# Note we set the name of the boot partition; we use that later to
# detect the disk has a partitioning scheme we support. See above.
cmdline = """parted -a optimal -ms %(device)s unit GiB \
mklabel gpt \
mkpart primary fat32 0%% %(boot_size)s \
set 1 boot on \
name 1 %(boot_label_name)s \
mkpart primary linux-swap %(boot_size)s %(swap_end)s \
mkpart primary ext4 %(swap_end)s %(scratch_end)s \
""" % dict(
boot_label_name=target._boot_label_name,
device=device,
boot_size=boot_size,
swap_end=boot_size + swap_size,
scratch_end=boot_size + swap_size + scratch_size,
)
offset = boot_size + swap_size + scratch_size
root_devs = [] # collect the root devices
pid = 4
while offset + root_size < size_gb:
cmdline += ' mkpart primary ext4 %d %d' % (offset, offset + root_size)
offset += root_size
root_devs.append(device_basename + target.kws['p_prefix'] + "%d" % pid)
pid += 1
target.shell.run(cmdline)
target.pos.fsinfo_read(target._boot_label_name)
# format quick
for root_dev in root_devs:
target.property_set('pos_root_' + root_dev, "EMPTY")
# now format filesystems
#
# note we only format the system boot partition (1), the linux
# swap(2) and the linux scratch space (3)
boot_dev = device + target.kws['p_prefix'] + "1"
swap_dev = device + target.kws['p_prefix'] + "2"
home_dev = device + target.kws['p_prefix'] + "3"
# Note: use FAT vs VFAT: vfat name translation creates issues when
# doing long file names; fat32 does not have that problem.
target.shell.run("mkfs.fat -F32 -n TCF-BOOT " + boot_dev)
target.shell.run("mkswap -L tcf-swap " + swap_dev)
target.shell.run("mkfs.ext4 -FqL tcf-scratch " + home_dev)
def _alloc_targets(rtb,
groups,
obo=None,
keepalive_period=4,
queue_timeout=None,
priority=700,
preempt=False,
queue=True,
reason=None,
wait_in_queue=True,
register_at=None):
"""
:param set register_at: (optional) if given, this is a set where
we will add the allocation ID created only if ACTIVE or QUEUED
inmediately as we get it before doing any waiting.
This is used for being able to cleanup on the exit path if the
client is cancelled.
"""
assert isinstance(groups, dict)
assert register_at == None or isinstance(register_at, set)
data = dict(
priority=priority,
preempt=preempt,
queue=queue,
groups={},
reason=reason,
)
if obo:
data['obo_user'] = obo
data['groups'] = groups
r = rtb.send_request("PUT", "allocation", json=data)
ts0 = time.time()
state = r['state']
if state not in ('queued', 'active'):
raise RuntimeError("allocation failed: %s: %s" %
(state, r.get('_message', 'message n/a')))
allocid = r['allocid']
if register_at != None: # if empty, it is ok
register_at.add(allocid)
data = {allocid: state}
if state == 'active': # got it
return allocid, state, r['group_allocated'].split(',')
if queue_timeout == 0:
return allocid, state, {}
ts = time.time()
group_allocated = []
commonl.progress(
"allocation ID %s: [+%.1fs] keeping alive during state '%s'" %
(allocid, ts - ts0, state))
new_state = state # in case we don't wait
while wait_in_queue:
if queue_timeout and ts - ts0 > queue_timeout:
raise tc.blocked_e(
"can't acquire targets, still busy after %ds" % queue_timeout,
dict(targets=groups))
time.sleep(keepalive_period)
ts = time.time()
state = data[allocid]
try:
r = rtb.send_request("PUT", "keepalive", json=data)
except requests.exceptions.RequestException:
# FIXME: tolerate N failures before giving up
pass
# COMPAT: old version packed the info in the 'result' field,
# newer have it in the first level dictionary
if 'result' in r:
result = r.pop('result')
r.update(result)
# COMPAT: end
commonl.progress(
"allocation ID %s: [+%.1fs] keeping alive during state '%s': %s" %
(allocid, ts - ts0, state, r))
if allocid not in r:
continue # no news
alloc = r[allocid]
new_state = alloc['state']
if new_state == 'active':
r = rtb.send_request("GET", "allocation/%s" % allocid)
group_allocated = r['group_allocated'].split(',')
break
elif new_state == 'invalid':
print "\nallocation ID %s: [+%.1fs] now invalid" % (allocid,
ts - ts0)
break
print "\nallocation ID %s: [+%.1fs] state transition %s -> %s" % (
allocid, ts - ts0, state, new_state)
data[allocid] = new_state
return allocid, new_state, group_allocated
