def _rootfs_guess(target, image, boot_dev): reason = "unknown issue" for tries in range(3): tries += 1 try: target.report_info("POS: guessing partition device [%d/3]" % tries) root_part_dev = _rootfs_guess_by_image(target, image, boot_dev) if root_part_dev != None: return root_part_dev # we couldn't find a root partition device, which means the # thing is trashed target.report_info("POS: repartitioning because couldn't find " "root partitions") _disk_partition(target) target.pos._fsinfo_load() except Exception as e: reason = str(e) if tries < 3: target.report_info("POS: failed to guess a root partition, " "retrying: %s" % reason) continue else: raise raise tc.blocked_e( "Tried too much to reinitialize the partition table to " "pick up a root partition? is there enough space to " "create root partitions?", dict(target = target, reason = reason, partsizes = target.kws.get('pos_partsizes', None)))
def _rootfs_guess(target, image, boot_dev): reason = "unknown issue" for tries in range(3): tries += 1 try: target.report_info("POS: guessing partition device [%d/3]" % tries) root_part_dev = _rootfs_guess_by_image(target, image, boot_dev) if root_part_dev != None: return root_part_dev # we couldn't find a root partition device, which means the # thing is trashed target.report_info("POS: repartitioning because couldn't find " "root partitions") _disk_partition(target) target.pos.fsinfo_read(target._boot_label_name) except Exception as e: reason = str(e) if tries < 3: target.report_info("POS: failed to guess a root partition, " "retrying: %s" % reason) continue else: raise raise tc.blocked_e( "Tried too much to reinitialize the partition table to " "pick up a root partition? is there enough space to " "create root partitions?", dict(target=target, reason=reason, partsizes=target.kws.get('pos_partsizes', None)))
def _linux_boot_guess_from_boot(target, image): """ Given a list of files (normally) in /boot, decide which ones are Linux kernels and initramfs; select the latest version """ # guess on the mounted filesystem, otherwise we get the POS! os_release = tl.linux_os_release_get(target, prefix="/mnt") distro = os_release.get('ID', None) output = target.shell.run("ls -1 /mnt/boot", output=True) kernel_regex = re.compile("(initramfs|initrd|bzImage|vmlinuz)(-(.*))?") kernel_versions = {} initramfs_versions = {} for line in output.split('\n'): m = kernel_regex.match(line) if not m: continue file_name = m.groups()[0] kver = m.groups()[1] if kver == None: kver = "default" if kver and ("rescue" in kver or "kdump" in kver): # these are usually found on Fedora continue elif file_name in ("initramfs", "initrd"): if kver.endswith(".img"): # remove .img extension that has been pegged to the version kver = os.path.splitext(kver)[0] initramfs_versions[kver] = line else: kernel_versions[kver] = line if len(kernel_versions) > 1 and 'default' in kernel_versions: del kernel_versions['default'] if len(kernel_versions) == 1: kver = kernel_versions.keys()[0] options = "" # image is atuple of (DISTRO, SPIN, VERSION, SUBVERSION, ARCH) if distro in ("fedora", "debian", "ubuntu") and 'live' in image: # Live distros needs this to boot, unknown exactly why; # also add console=tty0 to ensure it is not lost target.report_info("Linux Live hack: adding 'rw' to cmdline", dlevel=2) options = "console=tty0 rw" kernel = kernel_versions[kver] if kernel: kernel = "/boot/" + kernel initrd = initramfs_versions.get(kver, None) if initrd: initrd = "/boot/" + initrd return kernel, initrd, options elif len(kernel_versions) > 1: raise tc.blocked_e( "more than one Linux kernel in /boot; I don't know " "which one to use: " + " ".join(kernel_versions), dict(target=target, output=output)) else: return None, None, ""
def _linux_boot_guess_from_boot(target, image): """ Given a list of files (normally) in /boot, decide which ones are Linux kernels and initramfs; select the latest version """ # guess on the mounted filesystem, otherwise we get the POS! os_release = tl.linux_os_release_get(target, prefix = "/mnt") distro = os_release.get('ID', None) output = target.shell.run("ls -1 /mnt/boot", output = True) kernel_regex = re.compile("(initramfs|initrd|bzImage|vmlinuz)(-(.*))?") kernel_versions = {} initramfs_versions = {} for line in output.split('\n'): m = kernel_regex.match(line) if not m: continue file_name = m.groups()[0] kver = m.groups()[1] if kver == None: kver = "default" if kver and ("rescue" in kver or "kdump" in kver): # these are usually found on Fedora continue elif file_name in ( "initramfs", "initrd" ): if kver.endswith(".img"): # remove .img extension that has been pegged to the version kver = os.path.splitext(kver)[0] initramfs_versions[kver] = line else: kernel_versions[kver] = line if len(kernel_versions) > 1 and 'default' in kernel_versions: del kernel_versions['default'] if len(kernel_versions) == 1: kver = kernel_versions.keys()[0] options = "" # image is atuple of (DISTRO, SPIN, VERSION, SUBVERSION, ARCH) if distro in ("fedora", "debian", "ubuntu") and 'live' in image: # Live distros needs this to boot, unknown exactly why; # also add console=tty0 to ensure it is not lost target.report_info("Linux Live hack: adding 'rw' to cmdline", dlevel = 2) options = "console=tty0 rw" kernel = kernel_versions[kver] if kernel: kernel = "/boot/" + kernel initrd = initramfs_versions.get(kver, None) if initrd: initrd = "/boot/" + initrd return kernel, initrd, options elif len(kernel_versions) > 1: raise tc.blocked_e( "more than one Linux kernel in /boot; I don't know " "which one to use: " + " ".join(kernel_versions), dict(target = target, output = output)) else: return None, None, ""
def console_rx_poller(expecter, target, console = None): """ Poll a console """ # Figure out to which file we are writing console_id_name, console_code = console_mk_code(target, console) of = expecter.buffers.setdefault( console_code, open(os.path.join(target.testcase.buffersdir, "console-%s:%s-%s.log" % ( commonl.file_name_make_safe(target.fullid), target.kws['tc_hash'], console_id_name)), "a+", 0)) ofd = of.fileno() expecter.buffers.setdefault(console_code + "-ts0", time.time()) # Don't read too much, leave the rest for another run -- otherwise # we could spend our time trying to read a 1G console log file # from a really broken test case that spews a lot of stuff. # FIXME: move this to configuration max_size = 3000 # Read anything new since the last time we read -- this relies # on we having an exclusive lock on the target try: offset = os.fstat(ofd).st_size ts_start = time.time() target.report_info("reading from console %s @%d at %.2fs [%s]" % (console_id_name, offset, ts_start - expecter.ts0, of.name), dlevel = 3) # We are dealing with a file as our buffering and accounting # system, so because read_to_fd() is bypassing caching, flush # first and sync after the read. of.flush() total_bytes = target.rtb.rest_tb_target_console_read_to_fd( ofd, target.rt, console, offset, max_size, target.ticket) of.flush() os.fsync(ofd) ts_end = time.time() target.report_info("read from console %s @%d %dB at %.2fs (%.2fs) " "[%s]" % (console_id_name, offset, total_bytes, ts_end - expecter.ts0, ts_end - ts_start, of.name), dlevel = 3) # FIXME: do we want to print some debug of what we read? how # do we do it for huge files anyway? expecter._consoles.add(( target, console )) except requests.exceptions.HTTPError as e: raise tc.blocked_e("error reading console %s: %s\n" % (console_id_name, e), { "error trace": traceback.format_exc() }) # Don't count this as something that we need to treat as succesful return None
def console_rx_poller(expecter, target, console=None): """ Poll a console """ # Figure out to which file we are writing console_id_name, console_code = console_mk_code(target, console) of = expecter.buffers.setdefault( console_code, open( os.path.join( target.testcase.buffersdir, "console-%s:%s-%s.log" % (commonl.file_name_make_safe( target.fullid), target.kws['tc_hash'], console_id_name)), "a+", 0)) ofd = of.fileno() expecter.buffers.setdefault(console_code + "-ts0", time.time()) # Don't read too much, leave the rest for another run -- otherwise # we could spend our time trying to read a 1G console log file # from a really broken test case that spews a lot of stuff. # FIXME: move this to configuration max_size = 3000 # Read anything new since the last time we read -- this relies # on we having an exclusive lock on the target try: offset = os.fstat(ofd).st_size ts_start = time.time() target.report_info( "reading from console %s @%d at %.2fs [%s]" % (console_id_name, offset, ts_start - expecter.ts0, of.name), dlevel=3) # We are dealing with a file as our buffering and accounting # system, so because read_to_fd() is bypassing caching, flush # first and sync after the read. of.flush() total_bytes = target.rtb.rest_tb_target_console_read_to_fd( ofd, target.rt, console, offset, max_size, target.ticket) of.flush() os.fsync(ofd) ts_end = time.time() target.report_info("read from console %s @%d %dB at %.2fs (%.2fs) " "[%s]" % (console_id_name, offset, total_bytes, ts_end - expecter.ts0, ts_end - ts_start, of.name), dlevel=3) # FIXME: do we want to print some debug of what we read? how # do we do it for huge files anyway? expecter._consoles.add((target, console)) except requests.exceptions.HTTPError as e: raise tc.blocked_e( "error reading console %s: %s\n" % (console_id_name, e), {"error trace": traceback.format_exc()}) # Don't count this as something that we need to treat as succesful return None
def kbd_string_send(self, string, did="default_keyboard", press_time=0.2, interkey_time=0.2): """ Send a sequence of strings to a keyboard as key presses :param str string: string to send; note some characters can't be sent as is and have to be translated to key sequences (eg: press and hold shift, key X, relase shift). This is still not supported (FIXME) :param str did: (optional) name of the device to which to send it :param float press_time: (optional) time in seconds to press the key :param float interkey_time: (optional) time in seconds to wait between pressing keys for each string character """ assert isinstance(string, basestring) assert isinstance(did, basestring) assert isinstance(press_time, numbers.Real) assert isinstance(interkey_time, numbers.Real) device = self._device_get(did) s = "" for char in string.lower(): if char in self.key_mapping: key_code = self.key_mapping[char] else: raise tc.blocked_e( "still don't know how to send character '%s', add to" "input.key_mapping" % char) s += \ "%s EV_KEY %s 1\n" \ "WAIT %s\n" \ "%s EV_KEY %s 0 SYNC\n" \ "WAIT %s\n" \ % (device, key_code, press_time, device, key_code, interkey_time) self.sequence_send(s)
def _disk_partition(target): # we assume we are going to work on the boot device device_basename = target.kws['pos_boot_dev'] device = "/dev/" + device_basename target.shell.run('swapoff -a || true') # in case we autoswapped # find device size (FIXME: Linux specific) dev_info = None for blockdevice in target.pos.fsinfo.get('blockdevices', []): if blockdevice['name'] == device_basename: dev_info = blockdevice break else: raise tc.error_e( "%s: can't find information about this block device -- is " "the right pos_boot_device set in the configuration?" % device_basename, dict(fsinfo = pprint.pformat(target.pos.fsinfo))) size_gb = int(dev_info['size']) / 1024 / 1024 / 1024 target.report_info("POS: %s is %d GiB in size" % (device, size_gb), dlevel = 2) partsizes = target.kws.get('pos_partsizes', None) if partsizes == None: raise tc.blocked_e( "Can't partition target, it doesn't " "specify pos_partsizes tag", { 'target': target } ) partsize_l = partsizes.split(":") partsize_l = [ int(_partsize) for _partsize in partsize_l ] boot_size = partsize_l[0] swap_size = partsize_l[1] scratch_size = partsize_l[2] root_size = partsize_l[3] # note we set partition #0 as boot cmdline = """parted -a optimal -ms %(device)s unit GiB \ mklabel gpt \ mkpart primary fat32 0%% %(boot_size)s \ set 1 boot on \ mkpart primary linux-swap %(boot_size)s %(swap_end)s \ mkpart primary ext4 %(swap_end)s %(scratch_end)s \ """ % dict( device = device, boot_size = boot_size, swap_end = boot_size + swap_size, scratch_end = boot_size + swap_size + scratch_size, ) offset = boot_size + swap_size + scratch_size root_devs = [] # collect the root devices pid = 4 while offset + root_size < size_gb: cmdline += ' mkpart primary ext4 %d %d' % (offset, offset + root_size) offset += root_size root_devs.append(device_basename + target.kws['p_prefix'] + "%d" % pid) pid += 1 target.shell.run(cmdline) # Now set the root device information, so we can pick stuff to # format quick for root_dev in root_devs: target.property_set('pos_root_' + root_dev, "EMPTY") # Re-read partition tables target.shell.run('partprobe %s' % device) # now format filesystems # # note we only format the system boot partition (1), the linux # swap(2) and the linux scratch space (3) boot_dev = device + target.kws['p_prefix'] + "1" swap_dev = device + target.kws['p_prefix'] + "2" home_dev = device + target.kws['p_prefix'] + "3" # Note: use FAT vs VFAT: vfat name translation creates issues when # doing long file names; fat32 does not have that problem. target.shell.run("mkfs.fat -F32 -n TCF-BOOT " + boot_dev) target.shell.run("mkswap -L tcf-swap " + swap_dev) target.shell.run("mkfs.ext4 -FqL tcf-scratch " + home_dev)
def flash(self, images, upload=True, timeout=None, soft=False, hash_target_name=True): """Flash images onto target >>> target.images.flash({ >>> "kernel-86": "/tmp/file.bin", >>> "kernel-arc": "/tmp/file2.bin" >>> }, upload = True) or: >>> target.images.flash({ >>> "vmlinuz": "/tmp/vmlinuz", >>> "initrd": "/tmp/initrd" >>> }, upload = True) If *upload* is set to true, this function will first upload the images to the server and then flash them. :param dict images: dictionary keyed by (str) image type of things to flash in the target. e.g.: The types if images supported are determined by the target's configuration and can be reported with :meth:`list` (or command line *tcf images-ls TARGETNAME*). :param int timeout: (optional) seconds to wait for the operation to complete; defaults to whatever the interface declares in property *interfaces.images.IMAGETYPE.estimated_duration*. This is very tool and file specific, a bigger file with a slow tool is going to take way longer than a bigfer file on a slow one. :param bool upload: (optional) the image names are local files that need to be uploaded first to the server (this function will take care of that). :param boot soft: (optional, default *False*) if *True*, it will only flash an image if the hash of the file is different to the hash of the last image recorded in that image type (or if there is no record of anything having been flashed). """ if isinstance(images, dict): for k, v in images.items(): assert isinstance(k, basestring) \ and isinstance(v, basestring), \ "images has to be a dictionary IMAGETYPE:IMAGEFILE;" \ " all strings; %s, %s (%s, %s)" \ % (k, v, type(k), type(v)) else: raise AssertionError( "images has to be a dictionary IMAGETYPE:IMAGEFILE; got %s" \ % type(images)) target = self.target images_str = " ".join("%s:%s" % (k, v) for k, v in images.items()) if timeout == None: timeout = 0 for image_type, image in images.items(): images_data = target.rt['interfaces']['images'] image_data = images_data.get(image_type, None) if image_data == None: raise tc.blocked_e( "%s: image type '%s' not available" % (target.id, image_type), dict(target=target)) timeout += image_data.get("estimated_duration", 60) else: assert isinstance(timeout, int) # if we have to upload them, then we'll transform the names to # point to the names we got when uploading if upload: # Ok, we need to upload--the names in the dictionary point # to local filenames relative to the dir where we are # from, or absolute. Upload them to the server file space # for the user and give them a local name in there. _images = {} target.report_info("uploading: " + images_str, dlevel=2) for img_type, img_name in images.iteritems(): # the remote name will be NAME-DIGEST, so if multiple # testcases for the same user are uploading files with # the same name but different content / target, they don't # collide hd = commonl.hash_file_maybe_compressed( hashlib.sha512(), img_name) img_name_remote = \ hd[:10] \ + "-" + commonl.file_name_make_safe(os.path.abspath(img_name)) if hash_target_name: # put the target name first, otherwise we might # alter the extension that the server relies on to # autodecompress if need to img_name_remote = target.id + "-" + img_name_remote last_sha512 = target.rt['interfaces']['images']\ [img_type].get('last_sha512', None) if soft and last_sha512 == hd: # soft mode -- don't flash again if the last thing # flashed has the same hash as what we want to flash target.report_info( "%s:%s: skipping (soft flash: SHA512 match %s)" % (img_type, img_name, hd), dlevel=1) continue target.report_info("uploading: %s %s" % (img_type, img_name), dlevel=3) target.store.upload(img_name_remote, img_name) _images[img_type] = img_name_remote target.report_info("uploaded: %s %s" % (img_type, img_name), dlevel=2) target.report_info("uploaded: " + images_str, dlevel=1) else: _images = images if _images: # We don't do retries here, we leave it to the server target.report_info("flashing: " + images_str, dlevel=2) target.ttbd_iface_call("images", "flash", images=_images, timeout=timeout) target.report_info("flashed: " + images_str, dlevel=1) else: target.report_info("flash: all images soft flashed", dlevel=1)
def console_rx_eval(expecter, target, regex, console = None, _timeout = None, result = None, uid = None): """ Check what came on a console and act on it :param str uid: (optional) identifier to use to store offset data """ if hasattr(regex, "pattern"): what = regex.pattern else: what = regex regex = re.compile(re.escape(regex)) if not uid: uid = console_mk_uid(target, what, console, _timeout, result) console_id_name, console_code = console_mk_code(target, console) # These were set by the poller of = expecter.buffers.get(console_code, None) if of == None: # FIXME: debug lof output here? expecter->tc backlink? return None ofd = of.fileno() ts = time.time() # Get the offset we saved before as the last part where we looked # at. If none, then get the last offset the poller has # recorded. Otherwise, just default to look from the start # Note the idea is each eval function has a different offset where # it is looking at. Likewise the poller for each console. offset_poller_code = "offset_" + console_code offset = expecter.buffers_persistent.get( uid, expecter.buffers_persistent.get(offset_poller_code, 0)) if _timeout != False: timeout = expecter.timeout if _timeout == None else _timeout # We can do timeout checks that provide better information # than a generic 'timeout' if ts - expecter.ts0 > timeout: of.seek(offset) # so we report console from where searched raise tc.error_e( "expected console output '%s' from console '%s:%s' " \ "NOT FOUND after %.1f s" \ % (what, target.id, console_id_name, ts - expecter.ts0), { 'target': target, "console output": of }) # mmap the whole file (which doesn't alter the file pointer) # # We have to mmap as the file might be getting huge and thus, # reading line by line might be dumb. # # However, we only search starting at @offset, which is set later # to the last success searching we had. So we shan't really map # the whole file, shall map on demand. stat_info = os.fstat(ofd) if stat_info.st_size == 0: # Nothing to read return None with contextlib.closing(mmap.mmap(of.fileno(), 0, mmap.MAP_PRIVATE, mmap.PROT_READ, 0)) as mapping: target.report_info("looking for `%s` in console %s:%s @%d-%d at " "%.2fs [%s]" % (what, target.fullid, console_id_name, offset, stat_info.st_size, ts - expecter.ts0, of.name), dlevel = 3) m = regex.search(mapping[offset:]) if m: new_offset = offset + m.end() expecter.buffers_persistent[uid] = new_offset if result == None or result == "pass": # raising pass gets stopped at expecter.run(), so we # print instead, so we can see the console offset_tip = of.tell() # we report console from where searched of.seek(offset) # so we report console from where searched target.report_pass( "found expected `%s` in console `%s:%s` at %.2fs" % (what, target.fullid, console_id_name, ts - expecter.ts0), { "console output": of }, dlevel = 1, alevel = 2) of.seek(offset_tip) raise tc.pass_e( "found expected `%s` in console `%s:%s` at %.2fs" % (what, target.fullid, console_id_name, ts - expecter.ts0), {'target': target }) elif result == "fail": of.seek(offset) # so we report console from where searched raise tc.failed_e( "found expected (for failure) `%s` in console " "`%s:%s` at %.2fs" % (what, target.fullid, console_id_name, ts - expecter.ts0), { 'target': target, "console output": of }) elif result == "error" or result == "errr": of.seek(offset) # so we report console from where searched raise tc.error_e( "found expected (for error) `%s` in console " "`%s:%s` at %.2fs" % (what, target.fullid, console_id_name, ts - expecter.ts0), { 'target': target, "console output": of }) elif result == "skip": of.seek(offset) # so we report console from where searched raise tc.skip_e( "found expected (for skip) `%s` in console " "'%s:%s' at %.2fs" % (what, target.fullid, console_id_name, ts - expecter.ts0), { 'target': target, "console output": of }) else: of.seek(offset) # so we report console from where searched raise tc.blocked_e( "BUG: invalid result requested (%s)" % result) return None
def mount_fs(target, image, boot_dev): """ Boots a root filesystem on /mnt The partition used as a root filesystem is picked up based on the image that is going to be installed; we look for one that has the most similar image already installed and pick that. :returns: name of the root partition device """ pos_reinitialize = target.property_get("pos_reinitialize", False) if pos_reinitialize: # Need to reinit the partition table (we were told to by # setting pos_repartition to anything target.report_info("POS: repartitioning per pos_reinitialize " "property") for tag in target.rt.keys(): # remove pos_root_*, as they don't apply anymore if tag.startswith("pos_root_"): target.property_set(tag, None) _disk_partition(target) target.property_set('pos_reinitialize', None) root_part_dev = _rootfs_guess(target, image, boot_dev) # save for other functions called later target.root_part_dev = root_part_dev root_part_dev_base = os.path.basename(root_part_dev) image_prev = target.property_get("pos_root_" + root_part_dev_base, "nothing") target.report_info("POS: will use %s for root partition (had %s before)" % (root_part_dev, image_prev)) # fsinfo looks like described in target.pos._fsinfo_load() dev_info = None for blockdevice in target.pos.fsinfo.get('blockdevices', []): for child in blockdevice.get('children', []): if child['name'] == root_part_dev_base: dev_info = child if dev_info == None: # it cannot be we might have to repartition because at this # point *we* have partitoned. raise tc.error_e( "Can't find information for root device %s in FSinfo array" % root_part_dev_base, dict(fsinfo = target.pos.fsinfo)) # what format does it currently have? current_fstype = dev_info.get('fstype', 'ext4') # What format does it have to have? # # Ok, here we need to note that we can't have multiple root # filesystems with the same UUID or LABEL, so the image can't rely # on UUIDs # img_fss = target.pos.metadata.get('filesystems', {}) if '/' in img_fss: # a common origin is ok because the YAML schema forces both # fstype and mkfs_opts to be specified origin = "image's /.tcf.metadata.yaml" fsdata = img_fss.get('/', {}) else: origin = "defaults @" + commonl.origin_get(0) fsdata = {} fstype = fsdata.get('fstype', 'ext4') mkfs_opts = fsdata.get('mkfs_opts', '-Fj') # do they match? if fstype != current_fstype: target.report_info( "POS: reformatting %s because current format is '%s' and " "'%s' is needed (per %s)" % (root_part_dev, current_fstype, fstype, origin)) _mkfs(target, root_part_dev, fstype, mkfs_opts) else: target.report_info( "POS: no need to reformat %s because current format is '%s' and " "'%s' is needed (per %s)" % (root_part_dev, current_fstype, fstype, origin), dlevel = 1) for try_count in range(3): target.report_info("POS: mounting root partition %s onto /mnt " "to image [%d/3]" % (root_part_dev, try_count)) # don't let it fail or it will raise an exception, so we # print FAILED in that case to look for stuff; note the # double apostrophe trick so the regex finder doens't trip # on the command output = target.shell.run( "mount %s /mnt || echo FAI''LED" % root_part_dev, output = True) # What did we get? if 'FAILED' in output: if 'special device ' + root_part_dev \ + ' does not exist.' in output: _disk_partition(target) else: # ok, this probably means probably the partitions are not # formatted; so let's just reformat and retry _mkfs(target, root_part_dev, fstype, mkfs_opts) else: target.report_info("POS: mounted %s onto /mnt to image" % root_part_dev) return root_part_dev # it worked, we are done # fall through, retry else: raise tc.blocked_e( "POS: Tried to mount too many times and failed", dict(target = target))
def boot_config_multiroot(target, boot_dev, image): """ Configure the target to boot using the multiroot """ boot_dev = target.kws['pos_boot_dev'] # were we have mounted the root partition root_dir = "/mnt" linux_kernel_file, linux_initrd_file, linux_options = \ _linux_boot_guess(target, image) if linux_kernel_file == None: raise tc.blocked_e( "Cannot guess a Linux kernel to boot", dict(target = target)) # remove absolutization (some specs have it), as we need to copy from # mounted filesystems if os.path.isabs(linux_kernel_file): linux_kernel_file = linux_kernel_file[1:] if linux_initrd_file and os.path.isabs(linux_initrd_file): linux_initrd_file = linux_initrd_file[1:] if linux_options == None or linux_options == "": target.report_info("WARNING! can't figure out Linux cmdline " "options, taking defaults") # below we'll add more stuff linux_options = "console=tty0 root=SOMEWHERE" # MULTIROOT: indicate which image has been flashed to this # partition # saved by pos_multiroot.mountfs root_part_dev = target.root_part_dev root_part_dev_base = os.path.basename(root_part_dev) target.property_set('pos_root_' + root_part_dev_base, image) # /boot EFI system partition is always /dev/DEVNAME1 (first # partition), we partition like that # FIXME: we shouldn't harcode this boot_part_dev = boot_dev + target.kws['p_prefix'] + "1" kws = dict( boot_dev = boot_dev, boot_part_dev = boot_part_dev, root_part_dev = root_part_dev, root_part_dev_base = root_part_dev_base, root_dir = root_dir, linux_kernel_file = linux_kernel_file, linux_kernel_file_basename = os.path.basename(linux_kernel_file), linux_initrd_file = linux_initrd_file, linux_options = linux_options, ) if linux_initrd_file: kws['linux_initrd_file_basename'] = os.path.basename(linux_initrd_file) else: kws['linux_initrd_file_basename'] = None kws.update(target.kws) if linux_options: # # Maybe mess with the Linux boot options # target.report_info("linux cmdline options: %s" % linux_options) # FIXME: can this come from config? linux_options_replace = { # we want to use hard device name rather than LABELS/UUIDs, as # we have reformated and those will have changed "root": "/dev/%(root_part_dev_base)s" % kws, # we have created this in pos_multiroot and will only # replace it if the command line option is present. "resume": "/dev/disk/by-label/tcf-swap", } # FIXME: can this come from config? # We harcode a serial console on the device where we know the # framework is listening linux_options_append = \ "console=%(linux_serial_console_default)s,115200n8" % kws linux_options_append += " " + target.rt.get('linux_options_append', "") linux_options += " " + linux_options_append for option, value in linux_options_replace.iteritems(): regex = re.compile(r"\b" + option + r"=\S+") if regex.search(linux_options): linux_options = re.sub( regex, option + "=" + linux_options_replace[option], linux_options) else: linux_options += " " + option + "=" + value kws['linux_options'] = linux_options target.report_info("linux cmdline options (modified): %s" % linux_options) # Now generate the UEFI system partition that will boot the # system; we always override it, so we don't need to decide if it # is corrupted or whatever; we'll mount it in /boot (which now is # the POS /boot) # # Mount the /boot fs # # Try to assume it is ok, try to repair it if not; rsync the # kernels in there, it is faster for repeated operation/ # target.report_info("POS/EFI: checking %(boot_part_dev)s" % kws) output = target.shell.run( "fsck.fat -aw /dev/%(boot_part_dev)s || true" % kws, output = True, trim = True) # FIXME: parse the output to tell if there was a problem; when bad # but recovered, we'll see # # 0x41: Dirty bit is set. Fs was not properly unmounted and some data may be corrupt. # Automatically removing dirty bit. # Performing changes. # /dev/sda1: 11 files, 4173/261372 clusters # When ok # ## $ sudo fsck.vfat -wa /dev/nvme0n1p1 ## fsck.fat 4.1 (2017-01-24) ## /dev/sda1: 39 files, 2271/33259 clusters # When really hosed it won't print the device line, so we look for # that # ## $ fsck.vfat -wa /dev/trash ## fsck.fat 4.1 (2017-01-24) ## Logical sector size (49294 bytes) is not a multiple of the physical sector size. good_regex = re.compile("^/dev/%(boot_part_dev)s: [0-9]+ files, " "[0-9]+/[0-9]+ clusters$" % kws, re.MULTILINE) if not good_regex.search(output): target.report_info( "POS/EFI: /dev/%(boot_part_dev)s: formatting EFI " "filesystem, fsck couldn't fix it" % kws, dict(output = output)) target.shell.run("mkfs.fat -F32 /dev/%(boot_part_dev)s; sync" % kws) target.report_info( "POS/EFI: /dev/%(boot_part_dev)s: mounting in /boot" % kws) target.shell.run(" mount /dev/%(boot_part_dev)s /boot; " " mkdir -p /boot/loader/entries " % kws) # Do we have enough space? if not, remove the oldest stuff that is # not the file we are looking for # This prints ## $ df --output=pcent /boot ## Use% ## 6% output = target.shell.run("df --output=pcent /boot", output = True) regex = re.compile(r"^\s*(?P<percent>[\.0-9]+)%$", re.MULTILINE) match = regex.search(output) if not match: raise tc.error_e("Can't determine the amount of free space in /boot", dict(output = output)) used_space = float(match.groupdict()['percent']) if used_space > 75: target.report_info( "POS/EFI: /dev/%(boot_part_dev)s: freeing up space" % kws) # List files in /boot, sort by last update (when we rsynced # them) ## 2018-10-29+08:48:48.0000000000 84590 /boot/EFI/BOOT/BOOTX64.EFI ## 2018-10-29+08:48:48.0000000000 84590 /boot/EFI/systemd/systemd-bootx64.efi ## 2019-05-14+13:25:06.0000000000 7192832 /boot/vmlinuz-4.12.14-110-default ## 2019-05-14+13:25:08.0000000000 9688340 /boot/initrd-4.12.14-110-default ## 2019-05-14+13:25:14.0000000000 224 /boot/loader/entries/tcf-boot.conf ## 2019-05-14+13:25:14.0000000000 54 /boot/loader/loader.conf output = target.shell.run( # that double \\ needed so the shell is the one using it # as a \t, not python converting to a sequence "find /boot/ -type f -printf '%T+\\t%s\\t%p\\n' | sort", output = True, trim = True) # delete the first half entries over 300k except those that # match the kernels we are installing to_remove = [] _linux_initrd_file = kws.get("linux_initrd_file", "%%NONE%%") if linux_initrd_file == None: _linux_initrd_file = "%%NONE%%" for line in output.splitlines(): _timestamp, size_s, path = line.split(None, 3) size = int(size_s) if size > 300 * 1024 \ and not kws['linux_kernel_file_basename'] in path \ and not _linux_initrd_file in path: to_remove.append(path) # get older half and wipe it--this means the first half, as we # sort from older to newer to_remove = to_remove[:len(to_remove)//2] for path in to_remove: # we could do them in a single go, but we can exceed the # command line length -- lazy to optimize it target.shell.run("rm -f %s" % path) # Now copy all the files needed to boot to the root of the EFI # system partition mounted in /boot; remember they are in /mnt/, # our root partition target.shell.run( "time -p rsync --force --inplace /mnt/%(linux_kernel_file)s" " /boot/%(linux_kernel_file_basename)s" % kws) if kws.get("linux_initrd_file", None): target.shell.run( "time -p rsync --force --inplace /mnt/%(linux_initrd_file)s" " /boot/%(linux_initrd_file_basename)s" % kws) # we are the only one who cuts the cod here (yeah, direct Spanish # translation for the reader's entertainment), and if not wipe'm to # eternity; otherwise systemd will boot something prolly in # alphabetical order, not what we want target.shell.run("/usr/bin/rm -rf /boot/loader/entries/*.conf") # remember paths to the bootloader are relative to /boot # merge these two tcf_boot_conf = """\ cat <<EOF > /boot/loader/entries/tcf-boot.conf title TCF-driven local boot linux /%(linux_kernel_file_basename)s options %(linux_options)s """ if kws.get("linux_initrd_file", None): tcf_boot_conf += "initrd /%(linux_initrd_file_basename)s\n" tcf_boot_conf += "EOF\n" target.shell.run(tcf_boot_conf % kws) # Install new -- we wiped the /boot fs new anyway; if there are # multiple options already, bootctl shall be able to handle it. # Don't do variables in the any casewe will poke with them later # on anyway. Why? Because there is space for a race condition that # will leave us with the system booting off the localdisk vs the # network for PXE--see efibootmgr_setup() target.shell.run("bootctl update --no-variables" " || bootctl install --no-variables;" " sync") # Now mess with the EFIbootmgr # FIXME: make this a function and a configuration option (if the # target does efibootmgr) _efibootmgr_setup(target, boot_dev, 1) # umount only if things go well # Shall we try to unmount in case of error? nope, we are going to # have to redo the whole thing anyway, so do not touch it, in case # we are jumping in for manual debugging target.shell.run("umount /dev/%(boot_part_dev)s" % kws)
def _complete(self, reporter, runid, hashid, tc_name, doc): # Deliver to mongodb after adding a few more fields doc['runid'] = runid doc['hashid'] = hashid doc['tc_name'] = tc_name doc['timestamp'] = datetime.datetime.utcnow() if runid: doc['_id'] = runid + ":" + hashid else: doc['_id'] = hashid doc['target_name'] = reporter.target_group.name \ if reporter.target_group else 'n/a' if reporter.targets: servers = set() # We don't care about reps in servers target_types = [] # Here we want one per target doc['targets'] = {} # Note this is sorted by target-want-name, the names # assigned by the testcase to the targets, so all the # types and server lists are sorted by that. for tgname in sorted(reporter.targets.keys()): target = reporter.targets[tgname] doc['targets'][tgname] = dict(server=target.rtb.aka, id=target.id, type=target.type, bsp_model=target.bsp_model) servers.add(target.rtb.aka) if len(target.rt.get('bsp_models', {})) > 1: target_types.append(target.type + ":" + target.bsp_model) else: target_types.append(target.type) doc['target_types'] = ",".join(target_types) doc['target_servers'] = ",".join(servers) else: doc['target_types'] = 'static' doc['target_servers'] = 'static' tags = {} components = [] for tag in reporter._tags: (value, _origin) = reporter.tag_get(tag, "", "") tags[tag] = str(value) if tag == 'components': components = value.split() doc['tags'] = tags doc['components'] = components for complete_hook in self.complete_hooks: complete_hook(_tc, runid, hashid, tc_name, doc) retry_count = 1 while retry_count <= 3: if not self.results or self.made_in_pid != os.getpid(): self._mongo_setup() try: # We replace any existing reports for this _id -- if that # is not to happen, provide a different runid... self.results.find_one_and_replace({'_id': doc['_id']}, doc, upsert=True) break except pymongo.errors.PyMongoError as e: if retry_count <= 3: raise tc.blocked_e( "MongoDB error, can't record result: %s" % e) else: self.results = None reporter.warning("MongoDB error, reconnecting (%d/3): %s" % (e, retry_count))
def _linux_boot_guess_from_grub_cfg(target, _image): """ Extract kernel, initrd, kernel args from grub configuration Parses a grub config file to extract which boot entries are available, with their kernel and initrd files, the command line options and the manu entry names. Eliminate recovery and default, we shall be left with only one that is the main boot option, the one that always has to be booted that we will replicate. A grub.cfg looks like a lot of configuration stuff, but at the end, it boils down to entries like:: menuentry 'SLED 15-SP1' --class sled --class gnu-linux --class gnu --class os $menuentry_id_option 'gnulinux-simple-6bfcca25-0f4b-4d6c-87a0-c3bb25e5d512' { load_video set gfxpayload=keep insmod gzio insmod part_gpt insmod btrfs set root='hd0,gpt2' if [ x$feature_platform_search_hint = xy ]; then search --no-floppy --fs-uuid --set=root --hint-bios=hd0,gpt2 --hint-efi=hd0,gpt2 --hint-baremetal=ahci0,gpt2 --hint='hd0,gpt2' 6bfcca25-0f4b-4d6c-87a0-c3bb25e5d512 else search --no-floppy --fs-uuid --set=root 6bfcca25-0f4b-4d6c-87a0-c3bb25e5d512 fi echo 'Loading Linux 4.12.14-110-default ...' linux /boot/vmlinuz-4.12.14-110-default root=UUID=6bfcca25-0f4b-4d6c-87a0-c3bb25e5d512 ${extra_cmdline} splash=silent resume=/dev/disk/by-id/ata-QEMU_HARDDISK_QM00001-part3 quiet crashkernel=173M,high echo 'Loading initial ramdisk ...' initrd /boot/initrd-4.12.14-110-default } There is also the older style:: default=0 timeout=6 splashimage=/grub/android-x86.xpm.gz root (hd0,0) title Android-x86 7.1-r2 kernel /android-7.1-r2/kernel quiet root=/dev/ram0 androidboot.selinux=permissive vmalloc=192M buildvariant=userdebug SRC=/android-7.1-r2 initrd /android-7.1-r2/initrd.img title Android-x86 7.1-r2 (Debug mode) kernel /android-7.1-r2/kernel root=/dev/ram0 androidboot.selinux=permissive vmalloc=192M buildvariant=userdebug DEBUG=2 SRC=/android-7.1-r2 initrd /android-7.1-r2/initrd.img title Android-x86 7.1-r2 (Debug nomodeset) kernel /android-7.1-r2/kernel nomodeset root=/dev/ram0 androidboot.selinux=permissive vmalloc=192M buildvariant=userdebug DEBUG=2 SRC=/android-7.1-r2 initrd /android-7.1-r2/initrd.img title Android-x86 7.1-r2 (Debug video=LVDS-1:d) kernel /android-7.1-r2/kernel video=LVDS-1:d root=/dev/ram0 androidboot.selinux=permissive vmalloc=192M buildvariant=userdebug DEBUG=2 SRC=/android-7.1-r2 initrd /android-7.1-r2/initrd.img We ignore all but menuentry, linux, initrd; initrd might be missing """ # ignore errors if it does not exist grub_cfg_path = target.shell.run( # /mnt/grub|menu.lst is android ISO x86 # /mnt/boot|grub.cfg most other distros # ignore failures if the find process errors out "find /mnt/grub /mnt/boot" # redirect erros to /dev/null so they don't pollute the output we need " -iname grub.cfg -o -iname menu.lst 2> /dev/null" " || true", output=True, trim=True) grub_cfg_path = grub_cfg_path.strip() if grub_cfg_path == "": # no grub.cfg to rely on target.report_info("POS/uefi: found no grub config") return None, None, None target.report_info("POS/uefi: found grub config at %s" % grub_cfg_path) # read grub in, but only the parts we care for--it's faster grub_cfg = target.shell.run( r" grep --color=never -w '\(menuentry\|title\|kernel\|linux\|initrd\)' %s" % grub_cfg_path, output=True, trim=True) # a grub entry class _entry(object): name = None linux = None linux_args = None initrd = None # we use a dictionary to remove duplicate entries # # we key the entries by the initrd, linux kernel and kernel args they # use; if they are the same, it will be overriden and we won't have to # pick one. target._grub_entries = {} entry = _entry() def _entry_record(): entry_id = \ (entry.linux if entry.linux else "" ) \ + (entry.initrd if entry.initrd else "") \ + (entry.linux_args if entry.linux_args else "") if entry_id != "": # record existing target._grub_entries[entry_id] = entry entry_count = 0 for line in grub_cfg.splitlines(): # match menuentry lines and save name, flushing previous data # if any m = _grub_menuentry_regex.search(line) if m: # new entry!, close the previous _entry_record() entry = _entry() gd = m.groupdict() # the entries will always exist because that's the regex # search output, but we need to check they are not empty if gd['name1']: entry.name = gd['name1'] elif gd['name2']: entry.name = gd['name2'] else: entry.name = "entry #%d" % entry_count entry_count += 1 continue # match linux/initrd lines and extract values m = _grub_linux_initrd_entry_regex.search(line) if not m: continue gd = m.groupdict() linux = gd.get('linux', None) if linux: entry.linux = linux linux_args = gd.get('linux_args', None) if linux: # remove any ${SUBST} from the command line linux_args = re.sub(_grub_var_regex, "", linux_args) entry.linux_args = linux_args initrd = gd.get('initrd', None) if initrd: entry.initrd = initrd _entry_record() # record last entry # delete recovery / rescue stuff, we don't use it for entry_id in list(target._grub_entries.keys()): entry = target._grub_entries[entry_id] if _ignore_boot_entry(target, entry.name, grub_cfg_path.replace("/mnt", "")): del target._grub_entries[entry_id] if len(target._grub_entries) > 1: entries = pprint.pformat( [i.__dict__ for i in target._grub_entries.values()]) raise tc.blocked_e( "more than one Linux kernel entry; I don't know " "which one to use", dict(target=target, entries=entries)) if not target._grub_entries: # can't find? del target._grub_entries # need no more return None, None, None entry = target._grub_entries.values()[0] del target._grub_entries # need no more # note we assume the grub.cfg entries are in [/mnt]/boot because # grub.cfg is in /boot and there is usually a filesystem just for # /boot, thus the entries are relative. if entry.linux: entry.linux = "/boot/" + entry.linux if initrd: entry.initrd = "/boot/" + entry.initrd target.report_info( "POS/EFI: %s: scanning found kernel %s initrd %s args %s" % (grub_cfg_path, entry.linux, entry.initrd, entry.linux_args)) return entry.linux, entry.initrd, entry.linux_args
def _linux_boot_guess_from_lecs(target, _image): """ Setup a Linux kernel to boot using Gumniboot """ # ignore errors if it does not exist lecs = target.shell.run( r"find /mnt/boot/loader/entries -type f -iname \*.conf || true", output = True) # this returns something like # # find /mnt/boot/loader/entries -type f -iname \*.conf # /mnt/boot/loader/entries/Clear-linux-native-4.18.13-644.conf # /mnt/boot/loader/entries/Something-else.conf # 10 $ # # Filter just the output we care for lecl = [] for lec in lecs.split("\n"): lec = lec.strip() if not lec.startswith("/mnt/boot/loader/entries/"): continue if _ignore_boot_entry(target, os.path.basename(lec), os.path.dirname(lec)): continue lecl.append(lec) target.report_info("Loader Entry found: %s" % lec, dlevel = 1) if len(lecl) > 1: raise tc.blocked_e( "multiple loader entries in /boot, do not " "know which one to use: " + " ".join(lecl), dict(target = target)) elif len(lecl) == 0: return None, None, None # fallthrough, only one entry lec = lecl[0] output = target.shell.run('cat %s' % lec, output = True) kernel = None initrd = None options = None # read a loader entry, extract the kernel, initramfs and options # thanks Loader Entry Specification for making them single liners... dibs_regex = re.compile(r"^\s*(?P<command>linux|initrd|efi|options)\s+" "(?P<value>[^\n]+)\n?") for line in output.splitlines(): m = dibs_regex.match(line) if not m: continue d = m.groupdict() command = d['command'] value = d['value'] if command == 'linux': kernel = value elif command == 'efi': kernel = value elif command == 'initrd': initrd = value elif command == 'options': options = value # note we assume the LEC entries are in [/mnt]/boot because LEC # specifies them relateive to the filesystem if kernel: kernel = "/boot/" + kernel if initrd: initrd = "/boot/" + initrd return kernel, initrd, options
def _disk_partition(target): # we assume we are going to work on the boot device device_basename = target.kws['pos_boot_dev'] device = "/dev/" + device_basename target.shell.run('swapoff -a || true') # in case we autoswapped # find device size (FIXME: Linux specific) dev_info = None for blockdevice in target.pos.fsinfo.get('blockdevices', []): if blockdevice['name'] == device_basename: dev_info = blockdevice break else: raise tc.error_e( "%s: can't find information about this block device -- is " "the right pos_boot_device set in the configuration?" % device_basename, dict(fsinfo=pprint.pformat(target.pos.fsinfo))) size_gb = int(dev_info['size']) / 1024 / 1024 / 1024 target.report_info("POS: %s is %d GiB in size" % (device, size_gb), dlevel=2) partsizes = target.kws.get('pos_partsizes', None) if partsizes == None: raise tc.blocked_e( "Can't partition target, it doesn't " "specify pos_partsizes tag", {'target': target}) partsize_l = partsizes.split(":") partsize_l = [int(_partsize) for _partsize in partsize_l] boot_size = partsize_l[0] swap_size = partsize_l[1] scratch_size = partsize_l[2] root_size = partsize_l[3] # note we set partition #0 as boot cmdline = """parted -a optimal -ms %(device)s unit GiB \ mklabel gpt \ mkpart primary fat32 0%% %(boot_size)s \ set 1 boot on \ mkpart primary linux-swap %(boot_size)s %(swap_end)s \ mkpart primary ext4 %(swap_end)s %(scratch_end)s \ """ % dict( device=device, boot_size=boot_size, swap_end=boot_size + swap_size, scratch_end=boot_size + swap_size + scratch_size, ) offset = boot_size + swap_size + scratch_size root_devs = [] # collect the root devices pid = 4 while offset + root_size < size_gb: cmdline += ' mkpart primary ext4 %d %d' % (offset, offset + root_size) offset += root_size root_devs.append(device_basename + target.kws['p_prefix'] + "%d" % pid) pid += 1 target.shell.run(cmdline) # Now set the root device information, so we can pick stuff to # format quick for root_dev in root_devs: target.property_set('pos_root_' + root_dev, "EMPTY") # Re-read partition tables target.shell.run('partprobe %s' % device) # now format filesystems # # note we only format the system boot partition (1), the linux # swap(2) and the linux scratch space (3) boot_dev = device + target.kws['p_prefix'] + "1" swap_dev = device + target.kws['p_prefix'] + "2" home_dev = device + target.kws['p_prefix'] + "3" # Note: use FAT vs VFAT: vfat name translation creates issues when # doing long file names; fat32 does not have that problem. target.shell.run("mkfs.fat -F32 -n TCF-BOOT " + boot_dev) target.shell.run("mkswap -L tcf-swap " + swap_dev) target.shell.run("mkfs.ext4 -FqL tcf-scratch " + home_dev)
def _linux_boot_guess_from_grub_cfg(target, _image): """ Extract kernel, initrd, kernel args from grub configuration Parses a grub config file to extract which boot entries are available, with their kernel and initrd files, the command line options and the manu entry names. Eliminate recovery and default, we shall be left with only one that is the main boot option, the one that always has to be booted that we will replicate. A grub.cfg looks like a lot of configuration stuff, but at the end, it boils down to entries like:: menuentry 'SLED 15-SP1' --class sled --class gnu-linux --class gnu --class os $menuentry_id_option 'gnulinux-simple-6bfcca25-0f4b-4d6c-87a0-c3bb25e5d512' { load_video set gfxpayload=keep insmod gzio insmod part_gpt insmod btrfs set root='hd0,gpt2' if [ x$feature_platform_search_hint = xy ]; then search --no-floppy --fs-uuid --set=root --hint-bios=hd0,gpt2 --hint-efi=hd0,gpt2 --hint-baremetal=ahci0,gpt2 --hint='hd0,gpt2' 6bfcca25-0f4b-4d6c-87a0-c3bb25e5d512 else search --no-floppy --fs-uuid --set=root 6bfcca25-0f4b-4d6c-87a0-c3bb25e5d512 fi echo 'Loading Linux 4.12.14-110-default ...' linux /boot/vmlinuz-4.12.14-110-default root=UUID=6bfcca25-0f4b-4d6c-87a0-c3bb25e5d512 ${extra_cmdline} splash=silent resume=/dev/disk/by-id/ata-QEMU_HARDDISK_QM00001-part3 quiet crashkernel=173M,high echo 'Loading initial ramdisk ...' initrd /boot/initrd-4.12.14-110-default } There is also the older style:: default=0 timeout=6 splashimage=/grub/android-x86.xpm.gz root (hd0,0) title Android-x86 7.1-r2 kernel /android-7.1-r2/kernel quiet root=/dev/ram0 androidboot.selinux=permissive vmalloc=192M buildvariant=userdebug SRC=/android-7.1-r2 initrd /android-7.1-r2/initrd.img title Android-x86 7.1-r2 (Debug mode) kernel /android-7.1-r2/kernel root=/dev/ram0 androidboot.selinux=permissive vmalloc=192M buildvariant=userdebug DEBUG=2 SRC=/android-7.1-r2 initrd /android-7.1-r2/initrd.img title Android-x86 7.1-r2 (Debug nomodeset) kernel /android-7.1-r2/kernel nomodeset root=/dev/ram0 androidboot.selinux=permissive vmalloc=192M buildvariant=userdebug DEBUG=2 SRC=/android-7.1-r2 initrd /android-7.1-r2/initrd.img title Android-x86 7.1-r2 (Debug video=LVDS-1:d) kernel /android-7.1-r2/kernel video=LVDS-1:d root=/dev/ram0 androidboot.selinux=permissive vmalloc=192M buildvariant=userdebug DEBUG=2 SRC=/android-7.1-r2 initrd /android-7.1-r2/initrd.img We ignore all but menuentry, linux, initrd; initrd might be missing """ # ignore errors if it does not exist grub_cfg_path = target.shell.run( # /mnt/grub|menu.lst is android ISO x86 # /mnt/boot|grub.cfg most other distros # ignore failures if the find process errors out "find /mnt/grub /mnt/boot" # redirect erros to /dev/null so they don't pollute the output we need " -iname grub.cfg -o -iname menu.lst 2> /dev/null" " || true", output = True, trim = True) grub_cfg_path = grub_cfg_path.strip() if grub_cfg_path == "": # no grub.cfg to rely on target.report_info("POS/uefi: found no grub config") return None, None, None target.report_info("POS/uefi: found grub config at %s" % grub_cfg_path) # read grub in, but only the parts we care for--it's faster grub_cfg = target.shell.run( r" grep --color=never -w '\(menuentry\|title\|kernel\|linux\|initrd\)' %s" % grub_cfg_path, output = True, trim = True) # a grub entry class _entry(object): name = None linux = None linux_args = None initrd = None # we use a dictionary to remove duplicate entries # # we key the entries by the initrd, linux kernel and kernel args they # use; if they are the same, it will be overriden and we won't have to # pick one. target._grub_entries = {} entry = _entry() def _entry_record(): entry_id = \ (entry.linux if entry.linux else "" ) \ + (entry.initrd if entry.initrd else "") \ + (entry.linux_args if entry.linux_args else "") if entry_id != "": # record existing target._grub_entries[entry_id] = entry entry_count = 0 for line in grub_cfg.splitlines(): # match menuentry lines and save name, flushing previous data # if any m = _grub_menuentry_regex.search(line) if m: # new entry!, close the previous _entry_record() entry = _entry() gd = m.groupdict() # the entries will always exist because that's the regex # search output, but we need to check they are not empty if gd['name1']: entry.name = gd['name1'] elif gd['name2']: entry.name = gd['name2'] else: entry.name = "entry #%d" % entry_count entry_count += 1 continue # match linux/initrd lines and extract values m = _grub_linux_initrd_entry_regex.search(line) if not m: continue gd = m.groupdict() linux = gd.get('linux', None) if linux: entry.linux = linux linux_args = gd.get('linux_args', None) if linux: # remove any ${SUBST} from the command line linux_args = re.sub(_grub_var_regex, "", linux_args) entry.linux_args = linux_args initrd = gd.get('initrd', None) if initrd: entry.initrd = initrd _entry_record() # record last entry # delete recovery / rescue stuff, we don't use it for entry_id in list(target._grub_entries.keys()): entry = target._grub_entries[entry_id] if _ignore_boot_entry(target, entry.name, grub_cfg_path.replace("/mnt", "")): del target._grub_entries[entry_id] if len(target._grub_entries) > 1: entries = pprint.pformat([ i.__dict__ for i in target._grub_entries.values() ]) raise tc.blocked_e( "more than one Linux kernel entry; I don't know " "which one to use", dict(target = target, entries = entries)) if not target._grub_entries: # can't find? del target._grub_entries # need no more return None, None, None entry = target._grub_entries.values()[0] del target._grub_entries # need no more return entry.linux, entry.initrd, entry.linux_args
def mount_fs(target, image, boot_dev): """ Boots a root filesystem on /mnt The partition used as a root filesystem is picked up based on the image that is going to be installed; we look for one that has the most similar image already installed and pick that. :returns: name of the root partition device """ # does the disk have a partition scheme we recognize? pos_partsizes = target.rt['pos_partsizes'] # the name we'll give to the boot partition; see # _disk_partition(); if we can't find it, we assume the things # needs to be repartitioned. Note it includes the sizes, so if we # change the sizes in the config it reformats automatically. # # TCF-multiroot-NN:NN:NN:NN target._boot_label_name = "TCF-multiroot-" + pos_partsizes pos_reinitialize_force = True boot_dev_base = os.path.basename(boot_dev) child = target.pos.fsinfo_get_child_by_partlabel(boot_dev_base, target._boot_label_name) if child: pos_reinitialize_force = False else: target.report_info("POS: repartitioning due to different" " partition schema") pos_reinitialize = target.property_get("pos_reinitialize", False) if pos_reinitialize: target.report_info("POS: repartitioning per pos_reinitialize " "property") if pos_reinitialize or pos_reinitialize_force: # Need to reinit the partition table (we were told to by # setting pos_repartition to anything or we didn't recognize # the existing partitioning scheme) for tag in target.rt.keys(): # remove pos_root_*, as they don't apply anymore if tag.startswith("pos_root_"): target.property_set(tag, None) _disk_partition(target) target.pos.fsinfo_read(target._boot_label_name) target.property_set('pos_reinitialize', None) root_part_dev = _rootfs_guess(target, image, boot_dev) # save for other functions called later target.root_part_dev = root_part_dev root_part_dev_base = os.path.basename(root_part_dev) image_prev = target.property_get("pos_root_" + root_part_dev_base, "nothing") target.report_info("POS: will use %s for root partition (had %s before)" % (root_part_dev, image_prev)) # fsinfo looks like described in target.pos.fsinfo_read() dev_info = None for blockdevice in target.pos.fsinfo.get('blockdevices', []): for child in blockdevice.get('children', []): if child['name'] == root_part_dev_base: dev_info = child if dev_info == None: # it cannot be we might have to repartition because at this # point *we* have partitioned. raise tc.error_e( "Can't find information for root device %s in FSinfo array" % root_part_dev_base, dict(fsinfo=target.pos.fsinfo)) # what format does it currently have? current_fstype = dev_info.get('fstype', 'ext4') # What format does it have to have? # # Ok, here we need to note that we can't have multiple root # filesystems with the same UUID or LABEL, so the image can't rely # on UUIDs # img_fss = target.pos.metadata.get('filesystems', {}) if '/' in img_fss: # a common origin is ok because the YAML schema forces both # fstype and mkfs_opts to be specified origin = "image's /.tcf.metadata.yaml" fsdata = img_fss.get('/', {}) else: origin = "defaults @" + commonl.origin_get(0) fsdata = {} fstype = fsdata.get('fstype', 'ext4') mkfs_opts = fsdata.get('mkfs_opts', '-Fj') # do they match? if fstype != current_fstype: target.report_info( "POS: reformatting %s because current format is '%s' and " "'%s' is needed (per %s)" % (root_part_dev, current_fstype, fstype, origin)) _mkfs(target, root_part_dev, fstype, mkfs_opts) else: target.report_info( "POS: no need to reformat %s because current format is '%s' and " "'%s' is needed (per %s)" % (root_part_dev, current_fstype, fstype, origin), dlevel=1) for try_count in range(3): target.report_info("POS: mounting root partition %s onto /mnt " "to image [%d/3]" % (root_part_dev, try_count)) # don't let it fail or it will raise an exception, so we # print FAILED in that case to look for stuff; note the # double apostrophe trick so the regex finder doens't trip # on the command output = target.shell.run("mount %s /mnt || echo FAI''LED" % root_part_dev, output=True) # What did we get? if 'FAILED' in output: if 'special device ' + root_part_dev \ + ' does not exist.' in output: _disk_partition(target) target.pos.fsinfo_read(target._boot_label_name) else: # ok, this probably means probably the partitions are not # formatted; so let's just reformat and retry _mkfs(target, root_part_dev, fstype, mkfs_opts) else: target.report_info("POS: mounted %s onto /mnt to image" % root_part_dev) return root_part_dev # it worked, we are done # fall through, retry else: raise tc.blocked_e("POS: Tried to mount too many times and failed", dict(target=target))
def flash_spec_parse(self, flash_image_s=None): """Parse a images to flash specification in a string (that might be taken from the environment The string describing what to flash is in the form:: [[no-]soft] [[no-]upload] IMAGE:NAME[ IMAGE:NAME[..]]] - *soft*: flash in soft mode (default *False) (see :meth:`target.images.flash <tcfl.target_ext_images.extension.flash>`) the image(s) will only be flashed if the image to be flashed is different than the last image that was flashed. - *upload*: flash in soft mode (default *True*) (see :meth:`target.images.flash <tcfl.target_ext_images.extension.flash>`). The file will be uploaded to the server first or it will be assumed it is already present in the server. - *IMAGETYPE:FILENAME* flash file *FILENAME* in flash destination *IMAGETYPE*; *IMAGETYPE* being any of the image destinations the target can flash; can be found with:: $ tcf image-ls TARGETNAME or from the inventory:: $ tcf get TARGETNAME -p interfaces.images The string specification will be taken, in this order from the following list - the *image_flash* parameter - environment *IMAGE_FLASH_<TYPE>* - environment *IMAGE_FLASH_<FULLID>* - environment *IMAGE_FLASH_<ID>* - environment *IMAGE_FLASH* With *TYPE*, *FULLID* and *ID* sanitized to any character outside of the ranges *[a-zA-Z0-9_]* replaced with an underscore (*_*). **Example** :: $ export IMAGE_FLASH_TYPE1="soft bios:path/to/bios.bin" $ tcf run test_SOMESCRIPT.py if *test_SOMESCRIPT.py* uses this template, every invocation of it on a machine of type TYPE1 will result on the file *path/to/bios.bin* being flashed on the *bios* location (however, because of *soft*, if it was already flashed before, it will be skipped). :return: a tuple of >>> ( DICT, UPLOAD, SOFT ) - *DICT*: Dictionary is a dictionary of IMAGETYPE/file name to flash: >>> { >>> IMGTYPE1: IMGFILE1, >>> IMGTYPE2: IMGFILE2, >>> ... >>> } - *UPLOAD*: boolean indicating if the user wants the files to be uploaded (*True*, default) or to assume they are already in the server (*False*). - *SOFT*: flash in soft mode (*True*) or not (*False*, default if not given). """ target = self.target if not flash_image_s: # empty, load from environment target_id_safe = commonl.name_make_safe( target.id, string.ascii_letters + string.digits) target_fullid_safe = commonl.name_make_safe( target.fullid, string.ascii_letters + string.digits) target_type_safe = commonl.name_make_safe( target.type, string.ascii_letters + string.digits) source = None # keep pylint happy sourcel = [ # go from most specifcy to most generic "IMAGE_FLASH_%s" % target_fullid_safe, "IMAGE_FLASH_%s" % target_id_safe, "IMAGE_FLASH_%s" % target_type_safe, "IMAGE_FLASH", ] for source in sourcel: flash_image_s = os.environ.get(source, None) if flash_image_s != None: break else: target.report_info( "skipping image flashing (no function argument nor environment: %s)" % " ".join(sourcel)) return {}, False, False else: source = "function argument" # verify the format if not self._image_flash_regex.search(flash_image_s): raise tc.blocked_e( "image specification in %s does not conform to the form" " [[no-]soft] [[no-]upload] IMAGE:NAME[ IMAGE:NAME[..]]]" % source, dict(target=target)) image_flash = {} soft = False upload = True for entry in flash_image_s.split(" "): if not entry: # empty spaces...welp continue if entry == "soft": soft = True continue if entry == "no-soft": soft = False continue if entry == "upload": upload = True continue if entry == "no-upload": upload = False continue name, value = entry.split(":", 1) image_flash[name] = value return image_flash, upload, soft
def console_rx_eval(expecter, target, regex, console=None, _timeout=None, result=None, uid=None): """ Check what came on a console and act on it :param str uid: (optional) identifier to use to store offset data """ if hasattr(regex, "pattern"): what = regex.pattern else: what = regex regex = re.compile(re.escape(regex)) if not uid: uid = console_mk_uid(target, what, console, _timeout, result) console_id_name, console_code = console_mk_code(target, console) # These were set by the poller of = expecter.buffers.get(console_code, None) if of == None: # FIXME: debug lof output here? expecter->tc backlink? return None ofd = of.fileno() ts = time.time() # Get the offset we saved before as the last part where we looked # at. If none, then get the last offset the poller has # recorded. Otherwise, just default to look from the start # Note the idea is each eval function has a different offset where # it is looking at. Likewise the poller for each console. offset_poller_code = "offset_" + console_code offset = expecter.buffers_persistent.get( uid, expecter.buffers_persistent.get(offset_poller_code, 0)) if _timeout != False: timeout = expecter.timeout if _timeout == None else _timeout # We can do timeout checks that provide better information # than a generic 'timeout' if ts - expecter.ts0 > timeout: of.seek(offset) # so we report console from where searched raise tc.error_e( "expected console output '%s' from console '%s:%s' " \ "NOT FOUND after %.1f s" \ % (what, target.id, console_id_name, ts - expecter.ts0), { 'target': target, "console output": of }) # mmap the whole file (which doesn't alter the file pointer) # # We have to mmap as the file might be getting huge and thus, # reading line by line might be dumb. # # However, we only search starting at @offset, which is set later # to the last success searching we had. So we shan't really map # the whole file, shall map on demand. stat_info = os.fstat(ofd) if stat_info.st_size == 0: # Nothing to read return None with contextlib.closing( mmap.mmap(of.fileno(), 0, mmap.MAP_PRIVATE, mmap.PROT_READ, 0)) as mapping: target.report_info("looking for `%s` in console %s:%s @%d-%d at " "%.2fs [%s]" % (what, target.fullid, console_id_name, offset, stat_info.st_size, ts - expecter.ts0, of.name), dlevel=3) m = regex.search(mapping[offset:]) if m: new_offset = offset + m.end() expecter.buffers_persistent[uid] = new_offset if result == None or result == "pass": # raising pass gets stopped at expecter.run(), so we # print instead, so we can see the console offset_tip = of.tell() # we report console from where searched of.seek(offset) # so we report console from where searched target.report_pass( "found expected `%s` in console `%s:%s` at %.2fs @%d" % (what, target.fullid, console_id_name, ts - expecter.ts0, offset + m.start()), {"console output": of}, dlevel=1, alevel=2) of.seek(offset_tip) raise tc.pass_e( "found expected `%s` in console `%s:%s` at %.2fs" % (what, target.fullid, console_id_name, ts - expecter.ts0), {'target': target}) elif result == "fail": of.seek(offset) # so we report console from where searched raise tc.failed_e( "found expected (for failure) `%s` in console " "`%s:%s` at %.2fs" % (what, target.fullid, console_id_name, ts - expecter.ts0), { 'target': target, "console output": of }) elif result == "error" or result == "errr": of.seek(offset) # so we report console from where searched raise tc.error_e( "found expected (for error) `%s` in console " "`%s:%s` at %.2fs" % (what, target.fullid, console_id_name, ts - expecter.ts0), { 'target': target, "console output": of }) elif result == "skip": of.seek(offset) # so we report console from where searched raise tc.skip_e( "found expected (for skip) `%s` in console " "'%s:%s' at %.2fs" % (what, target.fullid, console_id_name, ts - expecter.ts0), { 'target': target, "console output": of }) else: of.seek(offset) # so we report console from where searched raise tc.blocked_e("BUG: invalid result requested (%s)" % result) return None
def _linux_boot_guess_from_lecs(target, _image): """ Setup a Linux kernel to boot using Gumniboot """ # ignore errors if it does not exist lecs = target.shell.run( r"find /mnt/boot/loader/entries -type f -iname \*.conf || true", output=True) # this returns something like # # find /mnt/boot/loader/entries -type f -iname \*.conf # /mnt/boot/loader/entries/Clear-linux-native-4.18.13-644.conf # /mnt/boot/loader/entries/Something-else.conf # 10 $ # # Filter just the output we care for lecl = [] for lec in lecs.split("\n"): lec = lec.strip() if not lec.startswith("/mnt/boot/loader/entries/"): continue if _ignore_boot_entry(target, os.path.basename(lec), os.path.dirname(lec)): continue lecl.append(lec) target.report_info("Loader Entry found: %s" % lec, dlevel=1) if len(lecl) > 1: raise tc.blocked_e( "multiple loader entries in /boot, do not " "know which one to use: " + " ".join(lecl), dict(target=target)) elif len(lecl) == 0: return None, None, None # fallthrough, only one entry lec = lecl[0] output = target.shell.run('cat %s' % lec, output=True) kernel = None initrd = None options = None # read a loader entry, extract the kernel, initramfs and options # thanks Loader Entry Specification for making them single liners... dibs_regex = re.compile(r"^\s*(?P<command>linux|initrd|efi|options)\s+" "(?P<value>[^\n]+)\n?") for line in output.splitlines(): m = dibs_regex.match(line) if not m: continue d = m.groupdict() command = d['command'] value = d['value'] if command == 'linux': kernel = value elif command == 'efi': kernel = value elif command == 'initrd': initrd = value elif command == 'options': options = value # note we assume the LEC entries are in [/mnt]/boot because LEC # specifies them relateive to the filesystem if kernel: kernel = "/boot/" + kernel if initrd: initrd = "/boot/" + initrd return kernel, initrd, options
def boot_config_multiroot(target, boot_dev, image): """ Configure the target to boot using the multiroot """ boot_dev = target.kws['pos_boot_dev'] # were we have mounted the root partition root_dir = "/mnt" linux_kernel_file, linux_initrd_file, linux_options = \ _linux_boot_guess(target, image) if linux_kernel_file == None: raise tc.blocked_e("Cannot guess a Linux kernel to boot", dict(target=target)) # remove absolutization (some specs have it), as we need to copy from # mounted filesystems if os.path.isabs(linux_kernel_file): linux_kernel_file = linux_kernel_file[1:] if linux_initrd_file and os.path.isabs(linux_initrd_file): linux_initrd_file = linux_initrd_file[1:] if linux_options == None or linux_options == "": target.report_info("WARNING! can't figure out Linux cmdline " "options, taking defaults") # below we'll add more stuff linux_options = "console=tty0 root=SOMEWHERE" # MULTIROOT: indicate which image has been flashed to this # partition # saved by pos_multiroot.mountfs root_part_dev = target.root_part_dev root_part_dev_base = os.path.basename(root_part_dev) target.property_set('pos_root_' + root_part_dev_base, image) # /boot EFI system partition is always /dev/DEVNAME1 (first # partition), we partition like that # FIXME: we shouldn't harcode this boot_part_dev = boot_dev + target.kws['p_prefix'] + "1" kws = dict( boot_dev=boot_dev, boot_part_dev=boot_part_dev, root_part_dev=root_part_dev, root_part_dev_base=root_part_dev_base, root_dir=root_dir, linux_kernel_file=linux_kernel_file, linux_kernel_file_basename=os.path.basename(linux_kernel_file), linux_initrd_file=linux_initrd_file, linux_options=linux_options, ) if linux_initrd_file: kws['linux_initrd_file_basename'] = os.path.basename(linux_initrd_file) else: kws['linux_initrd_file_basename'] = None kws.update(target.kws) if linux_options: # # Maybe mess with the Linux boot options # target.report_info("linux cmdline options: %s" % linux_options) # FIXME: can this come from config? linux_options_replace = { # we want to use hard device name rather than LABELS/UUIDs, as # we have reformated and those will have changed "root": "/dev/%(root_part_dev_base)s" % kws, # we have created this in pos_multiroot and will only # replace it if the command line option is present. "resume": "/dev/disk/by-label/tcf-swap", } # FIXME: can this come from config? # We harcode a serial console on the device where we know the # framework is listening linux_options_append = \ "console=%(linux_serial_console_default)s,115200n8" % kws linux_options_append += " " + target.rt.get('linux_options_append', "") linux_options += " " + linux_options_append for option, value in linux_options_replace.iteritems(): regex = re.compile(r"\b" + option + r"=\S+") if regex.search(linux_options): linux_options = re.sub( regex, option + "=" + linux_options_replace[option], linux_options) else: linux_options += " " + option + "=" + value kws['linux_options'] = linux_options target.report_info("linux cmdline options (modified): %s" % linux_options) # Now generate the UEFI system partition that will boot the # system; we always override it, so we don't need to decide if it # is corrupted or whatever; we'll mount it in /boot (which now is # the POS /boot) # # Mount the /boot fs # # Try to assume it is ok, try to repair it if not; rsync the # kernels in there, it is faster for repeated operation/ # target.report_info("POS/EFI: checking %(boot_part_dev)s" % kws) output = target.shell.run("fsck.fat -aw /dev/%(boot_part_dev)s || true" % kws, output=True, trim=True) # FIXME: parse the output to tell if there was a problem; when bad # but recovered, we'll see # # 0x41: Dirty bit is set. Fs was not properly unmounted and some data may be corrupt. # Automatically removing dirty bit. # Performing changes. # /dev/sda1: 11 files, 4173/261372 clusters # When ok # ## $ sudo fsck.vfat -wa /dev/nvme0n1p1 ## fsck.fat 4.1 (2017-01-24) ## /dev/sda1: 39 files, 2271/33259 clusters # When really hosed it won't print the device line, so we look for # that # ## $ fsck.vfat -wa /dev/trash ## fsck.fat 4.1 (2017-01-24) ## Logical sector size (49294 bytes) is not a multiple of the physical sector size. good_regex = re.compile( "^/dev/%(boot_part_dev)s: [0-9]+ files, " "[0-9]+/[0-9]+ clusters$" % kws, re.MULTILINE) if not good_regex.search(output): target.report_info( "POS/EFI: /dev/%(boot_part_dev)s: formatting EFI " "filesystem, fsck couldn't fix it" % kws, dict(output=output)) target.shell.run("mkfs.fat -F32 /dev/%(boot_part_dev)s; sync" % kws) target.report_info("POS/EFI: /dev/%(boot_part_dev)s: mounting in /boot" % kws) target.shell.run(" mount /dev/%(boot_part_dev)s /boot" " && mkdir -p /boot/loader/entries " % kws) # Do we have enough space? if not, remove the oldest stuff that is # not the file we are looking for # This prints ## $ df --output=pcent /boot ## Use% ## 6% output = target.shell.run("df --output=pcent /boot", output=True) regex = re.compile(r"^\s*(?P<percent>[\.0-9]+)%$", re.MULTILINE) match = regex.search(output) if not match: raise tc.error_e("Can't determine the amount of free space in /boot", dict(output=output)) used_space = float(match.groupdict()['percent']) if used_space > 75: target.report_info( "POS/EFI: /dev/%(boot_part_dev)s: freeing up space" % kws) # List files in /boot, sort by last update (when we rsynced # them) ## 2018-10-29+08:48:48.0000000000 84590 /boot/EFI/BOOT/BOOTX64.EFI ## 2018-10-29+08:48:48.0000000000 84590 /boot/EFI/systemd/systemd-bootx64.efi ## 2019-05-14+13:25:06.0000000000 7192832 /boot/vmlinuz-4.12.14-110-default ## 2019-05-14+13:25:08.0000000000 9688340 /boot/initrd-4.12.14-110-default ## 2019-05-14+13:25:14.0000000000 224 /boot/loader/entries/tcf-boot.conf ## 2019-05-14+13:25:14.0000000000 54 /boot/loader/loader.conf output = target.shell.run( # that double \\ needed so the shell is the one using it # as a \t, not python converting to a sequence "find /boot/ -type f -printf '%T+\\t%s\\t%p\\n' | sort", output=True, trim=True) # delete the first half entries over 300k except those that # match the kernels we are installing to_remove = [] _linux_initrd_file = kws.get("linux_initrd_file", "%%NONE%%") if linux_initrd_file == None: _linux_initrd_file = "%%NONE%%" for line in output.splitlines(): _timestamp, size_s, path = line.split(None, 3) size = int(size_s) if size > 300 * 1024 \ and not kws['linux_kernel_file_basename'] in path \ and not _linux_initrd_file in path: to_remove.append(path) # get older half and wipe it--this means the first half, as we # sort from older to newer to_remove = to_remove[:len(to_remove) // 2] for path in to_remove: # we could do them in a single go, but we can exceed the # command line length -- lazy to optimize it target.shell.run("rm -f %s" % path) # Now copy all the files needed to boot to the root of the EFI # system partition mounted in /boot; remember they are in /mnt/, # our root partition target.shell.run( "time -p rsync --force --inplace /mnt/%(linux_kernel_file)s" " /boot/%(linux_kernel_file_basename)s" % kws) if kws.get("linux_initrd_file", None): target.shell.run( "time -p rsync --force --inplace /mnt/%(linux_initrd_file)s" " /boot/%(linux_initrd_file_basename)s" % kws) # we are the only one who cuts the cod here (yeah, direct Spanish # translation for the reader's entertainment), and if not wipe'm to # eternity; otherwise systemd will boot something prolly in # alphabetical order, not what we want target.shell.run("/usr/bin/rm -rf /boot/loader/entries/*.conf") # remember paths to the bootloader are relative to /boot # merge these two tcf_boot_conf = """\ cat <<EOF > /boot/loader/entries/tcf-boot.conf title TCF-driven local boot linux /%(linux_kernel_file_basename)s options %(linux_options)s """ if kws.get("linux_initrd_file", None): tcf_boot_conf += "initrd /%(linux_initrd_file_basename)s\n" tcf_boot_conf += "EOF\n" target.shell.run(tcf_boot_conf % kws) # Install new -- we wiped the /boot fs new anyway; if there are # multiple options already, bootctl shall be able to handle it. # Don't do variables in the any casewe will poke with them later # on anyway. Why? Because there is space for a race condition that # will leave us with the system booting off the localdisk vs the # network for PXE--see efibootmgr_setup() target.shell.run("bootctl update --no-variables" " || bootctl install --no-variables;" " sync") # Now mess with the EFIbootmgr # FIXME: make this a function and a configuration option (if the # target does efibootmgr) _efibootmgr_setup(target, boot_dev, 1) # umount only if things go well # Shall we try to unmount in case of error? nope, we are going to # have to redo the whole thing anyway, so do not touch it, in case # we are jumping in for manual debugging target.shell.run("umount /dev/%(boot_part_dev)s" % kws)
def _disk_partition(target): # we assume we are going to work on the boot device device_basename = target.kws['pos_boot_dev'] device = "/dev/" + device_basename target.shell.run('swapoff -a || true') # in case we autoswapped # find device size (FIXME: Linux specific) dev_info = target.pos.fsinfo_get_block(device_basename) size_gb = int(dev_info['size']) / 1024 / 1024 / 1024 target.report_info("POS: %s is %d GiB in size" % (device, size_gb), dlevel=2) partsizes = target.kws.get('pos_partsizes', None) if partsizes == None: raise tc.blocked_e( "Can't partition target, it doesn't " "specify pos_partsizes tag", {'target': target}) partsize_l = partsizes.split(":") partsize_l = [int(_partsize) for _partsize in partsize_l] boot_size = partsize_l[0] swap_size = partsize_l[1] scratch_size = partsize_l[2] root_size = partsize_l[3] # note we set partition #0 as boot # Note we set the name of the boot partition; we use that later to # detect the disk has a partitioning scheme we support. See above. cmdline = """parted -a optimal -ms %(device)s unit GiB \ mklabel gpt \ mkpart primary fat32 0%% %(boot_size)s \ set 1 boot on \ name 1 %(boot_label_name)s \ mkpart primary linux-swap %(boot_size)s %(swap_end)s \ mkpart primary ext4 %(swap_end)s %(scratch_end)s \ """ % dict( boot_label_name=target._boot_label_name, device=device, boot_size=boot_size, swap_end=boot_size + swap_size, scratch_end=boot_size + swap_size + scratch_size, ) offset = boot_size + swap_size + scratch_size root_devs = [] # collect the root devices pid = 4 while offset + root_size < size_gb: cmdline += ' mkpart primary ext4 %d %d' % (offset, offset + root_size) offset += root_size root_devs.append(device_basename + target.kws['p_prefix'] + "%d" % pid) pid += 1 target.shell.run(cmdline) target.pos.fsinfo_read(target._boot_label_name) # format quick for root_dev in root_devs: target.property_set('pos_root_' + root_dev, "EMPTY") # now format filesystems # # note we only format the system boot partition (1), the linux # swap(2) and the linux scratch space (3) boot_dev = device + target.kws['p_prefix'] + "1" swap_dev = device + target.kws['p_prefix'] + "2" home_dev = device + target.kws['p_prefix'] + "3" # Note: use FAT vs VFAT: vfat name translation creates issues when # doing long file names; fat32 does not have that problem. target.shell.run("mkfs.fat -F32 -n TCF-BOOT " + boot_dev) target.shell.run("mkswap -L tcf-swap " + swap_dev) target.shell.run("mkfs.ext4 -FqL tcf-scratch " + home_dev)
def _alloc_targets(rtb, groups, obo=None, keepalive_period=4, queue_timeout=None, priority=700, preempt=False, queue=True, reason=None, wait_in_queue=True, register_at=None): """ :param set register_at: (optional) if given, this is a set where we will add the allocation ID created only if ACTIVE or QUEUED inmediately as we get it before doing any waiting. This is used for being able to cleanup on the exit path if the client is cancelled. """ assert isinstance(groups, dict) assert register_at == None or isinstance(register_at, set) data = dict( priority=priority, preempt=preempt, queue=queue, groups={}, reason=reason, ) if obo: data['obo_user'] = obo data['groups'] = groups r = rtb.send_request("PUT", "allocation", json=data) ts0 = time.time() state = r['state'] if state not in ('queued', 'active'): raise RuntimeError("allocation failed: %s: %s" % (state, r.get('_message', 'message n/a'))) allocid = r['allocid'] if register_at != None: # if empty, it is ok register_at.add(allocid) data = {allocid: state} if state == 'active': # got it return allocid, state, r['group_allocated'].split(',') if queue_timeout == 0: return allocid, state, {} ts = time.time() group_allocated = [] commonl.progress( "allocation ID %s: [+%.1fs] keeping alive during state '%s'" % (allocid, ts - ts0, state)) new_state = state # in case we don't wait while wait_in_queue: if queue_timeout and ts - ts0 > queue_timeout: raise tc.blocked_e( "can't acquire targets, still busy after %ds" % queue_timeout, dict(targets=groups)) time.sleep(keepalive_period) ts = time.time() state = data[allocid] try: r = rtb.send_request("PUT", "keepalive", json=data) except requests.exceptions.RequestException: # FIXME: tolerate N failures before giving up pass # COMPAT: old version packed the info in the 'result' field, # newer have it in the first level dictionary if 'result' in r: result = r.pop('result') r.update(result) # COMPAT: end commonl.progress( "allocation ID %s: [+%.1fs] keeping alive during state '%s': %s" % (allocid, ts - ts0, state, r)) if allocid not in r: continue # no news alloc = r[allocid] new_state = alloc['state'] if new_state == 'active': r = rtb.send_request("GET", "allocation/%s" % allocid) group_allocated = r['group_allocated'].split(',') break elif new_state == 'invalid': print "\nallocation ID %s: [+%.1fs] now invalid" % (allocid, ts - ts0) break print "\nallocation ID %s: [+%.1fs] state transition %s -> %s" % ( allocid, ts - ts0, state, new_state) data[allocid] = new_state return allocid, new_state, group_allocated