def prepare_filesystems(self): self.images = os.path.join(self.workdir, '.images') os.makedirs(self.images) # The image for the boot partition. self.boot_images = [] volumes = self.gadget.volumes.values() assert len(volumes) == 1, 'For now, only one volume is allowed' volume = list(volumes)[0] for partnum, part in enumerate(volume.structures): part_img = os.path.join(self.images, 'part{}.img'.format(partnum)) self.boot_images.append(part_img) run('dd if=/dev/zero of={} count=0 bs={} seek=1'.format( part_img, part.size)) if part.filesystem is FileSystemType.vfat: # pragma: nobranch run('mkfs.vfat {}'.format(part_img)) # XXX: Does not handle the case of partitions at the end of the # image. next_avail = part.offset + part.size # The image for the root partition. # # XXX: Hard-codes 4GB image size. Hard-codes last sector for backup # GPT. avail_space = (4000000000 - next_avail - 4 * 1024) // MiB(1) if self.rootfs_size / MiB(1) > avail_space: # pragma: nocover raise AssertionError('No room for root filesystem data') self.rootfs_size = avail_space self.root_img = os.path.join(self.images, 'root.img') # create empty file with holes with open(self.root_img, "w"): pass os.truncate(self.root_img, avail_space * MiB(1)) # We defer creating the root file system image because we have to # populate it at the same time. See mkfs.ext4(8) for details. self._next.append(self.populate_filesystems)
def _mkfs_ext4(img_file, contents_dir, label='writable'): """Encapsulate the `mkfs.ext4` invocation. As of e2fsprogs 1.43.1, mkfs.ext4 supports a -d option which allows you to populate the ext4 partition at creation time, with the contents of an existing directory. Unfortunately, we're targeting Ubuntu 16.04, which has e2fsprogs 1.42.X without the -d flag. In that case, we have to sudo loop mount the ext4 file system and populate it that way. Which sucks because sudo. """ cmd = 'mkfs.ext4 -L {} -O -metadata_csum {} -d {}'.format( label, img_file, contents_dir) proc = run(cmd, check=False) if proc.returncode == 0: # pragma: notravis # We have a new enough e2fsprogs, so we're done. return run('mkfs.ext4 -L {} {}'.format(label, img_file)) # pragma: notravis # Only do this if the directory is non-empty. if not os.listdir(contents_dir): return with mount(img_file) as mountpoint: # pragma: notravis # fixme: everything is terrible. run('sudo cp -dR --preserve=mode,timestamps {}/* {}'.format( contents_dir, mountpoint), shell=True)
def mount(img): with ExitStack() as resources: # pragma: notravis tmpdir = resources.enter_context(TemporaryDirectory()) mountpoint = os.path.join(tmpdir, 'root-mount') os.makedirs(mountpoint) run('sudo mount -oloop {} {}'.format(img, mountpoint)) resources.callback(run, 'sudo umount {}'.format(mountpoint)) yield mountpoint
def _populate_one_volume(self, name, volume): for partnum, part in enumerate(volume.structures): part_img = volume.part_images[partnum] part_dir = os.path.join(volume.basedir, 'part{}'.format(partnum)) if part.role is StructureRole.system_data: # The root partition needs to be ext4, which may or may not be # populated at creation time, depending on the version of # e2fsprogs. mkfs_ext4(part_img, self.rootfs, self.args.cmd, part.filesystem_label, preserve_ownership=True) elif part.filesystem is FileSystemType.none: image = Image(part_img, part.size) offset = 0 for content in part.content: src = os.path.join(self.unpackdir, 'gadget', content.image) file_size = os.path.getsize(src) assert content.size is None or content.size >= file_size, ( 'Spec size {} < actual size {} of: {}'.format( content.size, file_size, content.image)) if content.size is not None: file_size = content.size # TODO: We need to check for overlapping images. if content.offset is not None: offset = content.offset end = offset + file_size if end > part.size: if part.name is None: if part.role is None: whats_wrong = part.type else: whats_wrong = part.role.value else: whats_wrong = part.name part_path = 'volumes:<{}>:structure:<{}>'.format( name, whats_wrong) self.exitcode = 1 raise DoesNotFit(partnum, part_path, end - part.size) image.copy_blob(src, bs=1, seek=offset, conv='notrunc') offset += file_size elif part.filesystem is FileSystemType.vfat: sourcefiles = SPACE.join( os.path.join(part_dir, filename) for filename in os.listdir(part_dir)) env = dict(MTOOLS_SKIP_CHECK='1') env.update(os.environ) run('mcopy -s -i {} {} ::'.format(part_img, sourcefiles), env=env) elif part.filesystem is FileSystemType.ext4: mkfs_ext4(part_img, part_dir, self.args.cmd, part.filesystem_label) else: raise AssertionError('Invalid part filesystem type: {}'.format( part.filesystem))
def test_run_fails_no_output(self): with ExitStack() as resources: log = resources.enter_context(LogCapture()) resources.enter_context( patch('ubuntu_image.helpers.subprocess_run', return_value=FakeProcNoOutput())) run('/bin/false') self.assertEqual(log.logs, [ (logging.ERROR, 'COMMAND FAILED: /bin/false'), ])
def _populate_one_volume(self, name, volume): for partnum, part in enumerate(volume.structures): part_img = volume.part_images[partnum] part_dir = os.path.join(volume.basedir, 'part{}'.format(partnum)) if part.role is StructureRole.system_data: # The root partition needs to be ext4, which may or may not be # populated at creation time, depending on the version of # e2fsprogs. mkfs_ext4(part_img, self.rootfs, self.args.cmd, part.filesystem_label, preserve_ownership=True) elif part.filesystem is FileSystemType.none: image = Image(part_img, part.size) offset = 0 for content in part.content: src = os.path.join(self.unpackdir, 'gadget', content.image) file_size = os.path.getsize(src) assert content.size is None or content.size >= file_size, ( 'Spec size {} < actual size {} of: {}'.format( content.size, file_size, content.image)) if content.size is not None: file_size = content.size # TODO: We need to check for overlapping images. if content.offset is not None: offset = content.offset end = offset + file_size if end > part.size: if part.name is None: if part.role is None: whats_wrong = part.type else: whats_wrong = part.role.value else: whats_wrong = part.name part_path = 'volumes:<{}>:structure:<{}>'.format( name, whats_wrong) self.exitcode = 1 raise DoesNotFit(partnum, part_path, end - part.size) image.copy_blob(src, bs=1, seek=offset, conv='notrunc') offset += file_size elif part.filesystem is FileSystemType.vfat: sourcefiles = SPACE.join( os.path.join(part_dir, filename) for filename in os.listdir(part_dir) ) env = dict(MTOOLS_SKIP_CHECK='1') env.update(os.environ) run('mcopy -s -i {} {} ::'.format(part_img, sourcefiles), env=env) elif part.filesystem is FileSystemType.ext4: mkfs_ext4(part_img, part_dir, self.args.cmd, part.filesystem_label) else: raise AssertionError('Invalid part filesystem type: {}'.format( part.filesystem))
def test_run(self): stderr = StringIO() with ExitStack() as resources: resources.enter_context( patch('ubuntu_image.helpers.sys.stderr', stderr)) resources.enter_context( patch('ubuntu_image.helpers.subprocess_run', return_value=FakeProc())) run('/bin/false') # stdout gets piped to stderr. self.assertEqual(stderr.getvalue(), 'COMMAND FAILED: /bin/falsefake stdoutfake stderr')
def populate_filesystems(self): # The boot file system is VFAT. sourcefiles = SPACE.join( os.path.join(self.bootfs, filename) for filename in os.listdir(self.bootfs) ) run('mcopy -i {} {} ::'.format(self.boot_img, sourcefiles), env=dict(MTOOLS_SKIP_CHECK='1')) # The root partition needs to be ext4, which can only be populated at # creation time. run('mkfs.ext4 {} -d {}'.format(self.root_img, self.rootfs)) self._next.append(self.make_disk)
def populate_rootfs_contents(self): dst = self.rootfs if self.args.filesystem: src = self.args.filesystem # 'cp -a' is faster than the python functions and makes sure all # meta information is preserved. run('cp -a {} {}'.format(os.path.join(src, '*'), dst), shell=True) else: src = os.path.join(self.unpackdir, 'chroot') for subdir in os.listdir(src): shutil.move(os.path.join(src, subdir), os.path.join(dst, subdir)) # Remove default grub bootloader settings as we ship bootloader bits # (binary blobs and grub.cfg) to a generated rootfs locally. grub_folder = os.path.join(dst, 'boot', 'grub') if os.path.exists(grub_folder): for file_name in os.listdir(grub_folder): file_path = os.path.join(grub_folder, file_name) if os.path.isdir(file_path): shutil.rmtree(file_path, ignore_errors=True) else: os.unlink(file_path) # Replace pre-defined LABEL in /etc/fstab with the one # we're using 'LABEL=writable' in grub.cfg. # TODO We need EFI partition in fstab too fstab_path = os.path.join(dst, 'etc', 'fstab') if os.path.exists(fstab_path): with open(fstab_path, 'r') as fstab: new_content = re.sub(r'(LABEL=)\S+', r'\1{}'.format(DEFAULT_FS_LABEL), fstab.read(), count=1) # Insert LABEL entry if it's not found at fstab fs_label = 'LABEL={}'.format(DEFAULT_FS_LABEL) if fs_label not in new_content: new_content += 'LABEL={} / {} defaults 0 0'.format( DEFAULT_FS_LABEL, DEFAULT_FS) with open(fstab_path, 'w') as fstab: fstab.write(new_content) if self.cloud_init is not None: # LP: #1633232 - Only write out meta-data when the --cloud-init # parameter is given. seed_dir = os.path.join(dst, 'var', 'lib', 'cloud', 'seed') cloud_dir = os.path.join(seed_dir, 'nocloud-net') os.makedirs(cloud_dir, exist_ok=True) metadata_file = os.path.join(cloud_dir, 'meta-data') with open(metadata_file, 'w', encoding='utf-8') as fp: print('instance-id: nocloud-static', file=fp) userdata_file = os.path.join(cloud_dir, 'user-data') shutil.copy(self.cloud_init, userdata_file) super().populate_rootfs_contents()
def _run_hook(self, name, path, env): _logger.debug('Running hook script at path {} for hook named ' '{}.'.format(path, name)) proc = run(path, check=False, env=env) # We handle the error separately as we want to raise our own exception. if proc.returncode != 0: raise HookError(name, path, proc.returncode, proc.stderr)
def set_parition_type(self, partnum, typecode): """Set the partition type for selected partition. Since libparted is unable to provide this functionality, we use sfdisk to be able to set arbitrary type identifiers. Please note that this method needs to be only used after all partition() operations have been performed. Any disk.commit() operation resets the type GUIDs to defaults. """ if isinstance(typecode, tuple): if self.schema is VolumeSchema.gpt: typecode = typecode[1] else: typecode = typecode[0] run(['sfdisk', '--part-type', self.path, str(partnum), str(typecode)])
def test_unsparse_swapfile_ext4(self): with ExitStack() as resources: tmpdir = resources.enter_context(TemporaryDirectory()) results_dir = os.path.join(tmpdir, 'results') os.makedirs(results_dir) # Empty swapfile. swapfile = os.path.join(results_dir, 'swapfile') run('dd if=/dev/zero of={} bs=10M count=1'.format(swapfile), stdout=DEVNULL, stderr=DEVNULL) # Image file. img_file = resources.enter_context(NamedTemporaryFile()) mock = MountMocker(results_dir, results_dir) resources.enter_context( patch('ubuntu_image.helpers.run', mock.run)) # Now the actual test. unsparse_swapfile_ext4(img_file) self.assertTrue(mock.dd_called)
def _calculate_dirsize(path): # more accruate way to calculate size of dir which # contains hard or soft links total = 0 proc = run('du -s -B1 {}'.format(path)) total = int(proc.stdout.strip().split()[0]) # Fudge factor for incidentals. total *= 1.5 return ceil(total)
def generate_manifests(self): # After the images are built, we would also like to have some image # manifests exported so that one can easily check what packages have # been installed on the rootfs. We utilize dpkg-query tool to generate # the manifest file for classic image. Packages like casper which is # only useful in a live CD/DVD are removed. # The deprecated words can be found below: # https://help.ubuntu.com/community/MakeALiveCD/DVD/BootableFlashFromHarddiskInstall deprecated_words = ['ubiquity', 'casper'] manifest_path = os.path.join(self.output_dir, 'filesystem.manifest') tmpfile_path = os.path.join(gettempdir(), 'filesystem.manifest') with open(tmpfile_path, 'w+') as tmpfile: query_cmd = ['sudo', 'chroot', self.rootfs, 'dpkg-query', '-W', '--showformat=${Package} ${Version}\n'] run(query_cmd, stdout=tmpfile, stderr=None, env=os.environ) tmpfile.seek(0, 0) with open(manifest_path, 'w') as manifest: for line in tmpfile: if not any(word in line for word in deprecated_words): manifest.write(line) super().generate_manifests()
def diagnostics(self): """Return diagnostics string. :return: Dictionary with disk parition information :rtype: dict """ status = run(['sfdisk', '--json', self.path]) disk_info = load_json(status.stdout) # TBD: # - check status # - log stderr return disk_info
def copy_blob(self, blob_path, **dd_args): """Copy a blob to the image file. The copy is done using ``dd`` for consistency. The keyword arguments are passed directly to the ``dd`` call. See the dd(1) manpage for details. :param blob_path: File system path to the input file. :type blob_path: str """ # Put together the dd command. args = ['dd', 'of={}'.format(self.path), 'if={}'.format(blob_path), 'conv=sparse'] for key, value in dd_args.items(): args.append('{}={}'.format(key, value)) # Run the command. We'll capture stderr for logging purposes. # # TBD: # - check status of the returned CompletedProcess # - handle errors # - log stdout/stderr run(args)
def populate_filesystems(self): volumes = self.gadget.volumes.values() assert len(volumes) == 1, 'For now, only one volume is allowed' volume = list(volumes)[0] for partnum, part in enumerate(volume.structures): part_img = self.boot_images[partnum] part_dir = os.path.join(self.workdir, 'part{}'.format(partnum)) if part.filesystem is FileSystemType.none: # pragma: nocover image = Image(part_img, part.size) offset = 0 for file in part.content: src = os.path.join(self.unpackdir, 'gadget', file.image) file_size = os.path.getsize(src) if file.size is not None and file.size < file_size: raise AssertionError('Size {} < size of {}'.format( file.size, file.image)) if file.size is not None: file_size = file.size # XXX: We need to check for overlapping images. if file.offset is not None: offset = file.offset # XXX: We must check offset+size vs. the target image. image.copy_blob(src, bs=1, seek=offset, conv='notrunc') offset += file_size elif part.filesystem is FileSystemType.vfat: # pragma: nobranch sourcefiles = SPACE.join( os.path.join(part_dir, filename) for filename in os.listdir(part_dir)) env = dict(MTOOLS_SKIP_CHECK='1') env.update(os.environ) run('mcopy -s -i {} {} ::'.format(part_img, sourcefiles), env=env) elif part.filesystem is FileSystemType.ext4: # pragma: nocover _mkfs_ext4(self.part_img, part_dir, part.filesystem_label) # The root partition needs to be ext4, which may or may not be # populated at creation time, depending on the version of e2fsprogs. _mkfs_ext4(self.root_img, self.rootfs) self._next.append(self.make_disk)
def prepare_filesystems(self): self.images = os.path.join(self._tmpdir, '.images') os.makedirs(self.images) # The image for the boot partition. self.boot_img = os.path.join(self.images, 'boot.img') run('dd if=/dev/zero of={} count=0 bs=1GB seek=1'.format( self.boot_img)) run('mkfs.vfat {}'.format(self.boot_img)) # The image for the root partition. self.root_img = os.path.join(self.images, 'root.img') run('dd if=/dev/zero of={} count=0 bs=1GB seek=2'.format( self.root_img)) # We defer creating the root file system image because we have to # populate it at the same time. See mkfs.ext4(8) for details. self._next.append(self.populate_filesystems)
def _populate_one_volume(self, name, volume): # For the LK bootloader we need to copy boot.img and snapbootsel.bin to # the gadget folder so they can be used as partition content. The first # one comes from the kernel snap, while the second one is modified by # 'snap prepare-image' to set the right core and kernel for the kernel # command line. if volume.bootloader is BootLoader.lk: boot = os.path.join(self.unpackdir, 'image', 'boot', 'lk') gadget = os.path.join(self.unpackdir, 'gadget') if os.path.isdir(boot): os.makedirs(gadget, exist_ok=True) for filename in os.listdir(boot): src = os.path.join(boot, filename) dst = os.path.join(gadget, filename) shutil.copy(src, dst) for partnum, part in enumerate(volume.structures): part_img = volume.part_images[partnum] # In seeded images, the system-seed partition is basically the # rootfs partition - at least from the ubuntu-image POV. if part.role is StructureRole.system_seed: part_dir = self.rootfs else: part_dir = os.path.join(volume.basedir, 'part{}'.format(partnum)) if part.role is StructureRole.system_data: # The root partition needs to be ext4, which may or may not be # populated at creation time, depending on the version of # e2fsprogs. mkfs_ext4(part_img, self.rootfs, self.args.cmd, part.filesystem_label, preserve_ownership=True) elif part.filesystem is FileSystemType.none: image = Image(part_img, part.size) offset = 0 for content in part.content: src = os.path.join(self.unpackdir, 'gadget', content.image) file_size = os.path.getsize(src) assert content.size is None or content.size >= file_size, ( 'Spec size {} < actual size {} of: {}'.format( content.size, file_size, content.image)) if content.size is not None: file_size = content.size # TODO: We need to check for overlapping images. if content.offset is not None: offset = content.offset end = offset + file_size if end > part.size: if part.name is None: if part.role is None: whats_wrong = part.type else: whats_wrong = part.role.value else: whats_wrong = part.name part_path = 'volumes:<{}>:structure:<{}>'.format( name, whats_wrong) self.exitcode = 1 raise DoesNotFit(partnum, part_path, end - part.size) image.copy_blob(src, bs=1, seek=offset, conv='notrunc') offset += file_size elif part.filesystem is FileSystemType.vfat: sourcefiles = SPACE.join( os.path.join(part_dir, filename) for filename in os.listdir(part_dir)) env = dict(MTOOLS_SKIP_CHECK='1') env.update(os.environ) run('mcopy -s -i {} {} ::'.format(part_img, sourcefiles), env=env) elif part.filesystem is FileSystemType.ext4: mkfs_ext4(part_img, part_dir, self.args.cmd, part.filesystem_label) else: raise AssertionError('Invalid part filesystem type: {}'.format( part.filesystem))
def _prepare_one_volume(self, volume_index, name, volume): volume.part_images = [] farthest_offset = 0 for partnum, part in enumerate(volume.structures): part_img = os.path.join(volume.basedir, 'part{}.img'.format(partnum)) # The system-data and system-seed partitions do not have to have # an explicit size set. if part.role in (StructureRole.system_data, StructureRole.system_seed): if part.size is None: part.size = self.rootfs_size elif part.size < self.rootfs_size: _logger.warning('rootfs partition size ({}) smaller than ' 'actual rootfs contents {}'.format( part.size, self.rootfs_size)) part.size = self.rootfs_size # Create the actual image files now. if part.role is StructureRole.system_data: # The image for the root partition. # We defer creating the root file system image because we have # to populate it at the same time. See mkfs.ext4(8) for # details. Path(part_img).touch() os.truncate(part_img, part.size) else: run('dd if=/dev/zero of={} count=0 bs={} seek=1'.format( part_img, part.size)) if part.filesystem is FileSystemType.vfat: label_option = ( '-n {}'.format(part.filesystem_label) # TODO: I think this could be None or the empty string, # but this needs verification. if part.filesystem_label else '') # TODO: hard-coding of sector size. run('mkfs.vfat -s 1 -S 512 -F 32 {} {}'.format( label_option, part_img)) volume.part_images.append(part_img) farthest_offset = max(farthest_offset, (part.offset + part.size)) # Calculate or check the final image size. # # TODO: Hard-codes last 34 512-byte sectors for backup GPT, # empirically derived from sgdisk behavior. calculated = ceil(farthest_offset / 1024 + 17) * 1024 if self.args.image_size is None: volume.image_size = calculated elif isinstance(self.args.image_size, int): # One size to rule them all. if self.args.image_size < calculated: _logger.warning('Ignoring image size smaller ' 'than minimum required size: vol[{}]:{} ' '{} < {}'.format(volume_index, name, self.args.given_image_size, calculated)) volume.image_size = calculated else: volume.image_size = self.args.image_size else: # The --image-size arguments are a dictionary, so look up the # one used for this volume. size_by_index = self.args.image_size.get(volume_index) size_by_name = self.args.image_size.get(name) if size_by_index is not None and size_by_name is not None: _logger.warning( 'Ignoring ambiguous volume size; index+name given') volume.image_size = calculated else: image_size = (size_by_index if size_by_name is None else size_by_name) if image_size < calculated: _logger.warning('Ignoring image size smaller ' 'than minimum required size: vol[{}]:{} ' '{} < {}'.format( volume_index, name, self.args.given_image_size, calculated)) volume.image_size = calculated else: volume.image_size = image_size
def _prepare_one_volume(self, volume_index, name, volume): volume.part_images = [] farthest_offset = 0 for partnum, part in enumerate(volume.structures): part_img = os.path.join( volume.basedir, 'part{}.img'.format(partnum)) if part.role is StructureRole.system_data: # The image for the root partition. if part.size is None: part.size = self.rootfs_size elif part.size < self.rootfs_size: _logger.warning('rootfs partition size ({}) smaller than ' 'actual rootfs contents {}'.format( part.size, self.rootfs_size)) part.size = self.rootfs_size # We defer creating the root file system image because we have # to populate it at the same time. See mkfs.ext4(8) for # details. Path(part_img).touch() os.truncate(part_img, part.size) else: run('dd if=/dev/zero of={} count=0 bs={} seek=1'.format( part_img, part.size)) if part.filesystem is FileSystemType.vfat: label_option = ( '-n {}'.format(part.filesystem_label) # TODO: I think this could be None or the empty string, # but this needs verification. if part.filesystem_label else '') # TODO: hard-coding of sector size. run('mkfs.vfat -s 1 -S 512 -F 32 {} {}'.format( label_option, part_img)) volume.part_images.append(part_img) farthest_offset = max(farthest_offset, (part.offset + part.size)) # Calculate or check the final image size. # # TODO: Hard-codes last 34 512-byte sectors for backup GPT, # empirically derived from sgdisk behavior. calculated = ceil(farthest_offset / 1024 + 17) * 1024 if self.args.image_size is None: volume.image_size = calculated elif isinstance(self.args.image_size, int): # One size to rule them all. if self.args.image_size < calculated: _logger.warning( 'Ignoring image size smaller ' 'than minimum required size: vol[{}]:{} ' '{} < {}'.format(volume_index, name, self.args.given_image_size, calculated)) volume.image_size = calculated else: volume.image_size = self.args.image_size else: # The --image-size arguments are a dictionary, so look up the # one used for this volume. size_by_index = self.args.image_size.get(volume_index) size_by_name = self.args.image_size.get(name) if size_by_index is not None and size_by_name is not None: _logger.warning( 'Ignoring ambiguous volume size; index+name given') volume.image_size = calculated else: image_size = (size_by_index if size_by_name is None else size_by_name) if image_size < calculated: _logger.warning( 'Ignoring image size smaller ' 'than minimum required size: vol[{}]:{} ' '{} < {}'.format(volume_index, name, self.args.given_image_size, calculated)) volume.image_size = calculated else: volume.image_size = image_size