def test_copy_blob_install_grub_to_mbr(self):
# Install GRUB to MBR
# TODO: this has to be represented in the image.yaml
# NOTE: the boot.img has to be a part of the gadget snap itself
# FIXME: embed a pointer to 2nd stage in bios-boot partition
#
# dd if=blobs/img.mbr of=img bs=446 count=1 conv=notrunc
#
# Start by creating a blob of the requested size.
blob_file = os.path.join(self.tmpdir, 'mbr.blob')
with open(blob_file, 'wb') as fp:
fp.write(b'happyhappyjoyjoy' * 27)
fp.write(b'happyhappyjoyj')
self.assertEqual(os.stat(blob_file).st_size, 446)
image = Image(self.img, MiB(1))
image.copy_blob(blob_file, bs=446, count=1, conv='notrunc')
# At the top of the image file, there should be 27 Stimpy
# Exclamations, followed by a happyhappyjoyj.
with open(image.path, 'rb') as fp:
complete_stimpys = fp.read(432)
partial_stimpys = fp.read(14)
# Spot check.
zeros = fp.read(108)
self.assertEqual(complete_stimpys, b'happyhappyjoyjoy' * 27)
self.assertEqual(partial_stimpys, b'happyhappyjoyj')
# Stevens $4.13 - the extended file should read as zeros.
self.assertEqual(zeros, b'\0' * 108)
def test_copy_blob_install_grub_to_mbr(self):
# Install GRUB to MBR
# TODO: this has to be represented in the gadget.yaml
# NOTE: the boot.img has to be a part of the gadget snap itself
# FIXME: embed a pointer to 2nd stage in bios-boot partition
#
# dd if=blobs/img.mbr of=img bs=446 count=1 conv=notrunc
#
# Start by creating a blob of the requested size.
blob_file = os.path.join(self.tmpdir, 'mbr.blob')
with open(blob_file, 'wb') as fp:
fp.write(b'happyhappyjoyjoy' * 27)
fp.write(b'happyhappyjoyj')
self.assertEqual(os.stat(blob_file).st_size, 446)
image = Image(self.img, MiB(1))
image.copy_blob(blob_file, bs=446, count=1, conv='notrunc')
# At the top of the image file, there should be 27 Stimpy
# Exclamations, followed by a happyhappyjoyj.
with open(image.path, 'rb') as fp:
complete_stimpys = fp.read(432)
partial_stimpys = fp.read(14)
# Spot check.
zeros = fp.read(108)
self.assertEqual(complete_stimpys, b'happyhappyjoyjoy' * 27)
self.assertEqual(partial_stimpys, b'happyhappyjoyj')
# Stevens $4.13 - the extended file should read as zeros.
self.assertEqual(zeros, b'\0' * 108)
def make_disk(self):
self.disk_img = os.path.join(self.images, 'disk.img')
image = Image(self.disk_img, GiB(4))
# Create BIOS boot partition
#
# The partition is 1MiB in size, as recommended by various
# partitioning guides. The actual required size is much, much
# smaller.
#
# https://www.gnu.org/software/grub/manual/html_node/BIOS-installation.html#BIOS-installation
# image.partition(new='1:4MiB:+1MiB')
# image.partition(typecode='1:21686148-6449-6E6F-744E-656564454649')
# image.partition(change_name='1:grub')
# image.copy_blob(self.boot_img,
# bs='1MiB', seek=4, count=1, conv='notrunc')
# Create EFI system partition
#
# TODO: switch to 512MiB as recommended by the standard
image.partition(new='2:5MiB:+64MiB')
image.partition(typecode='2:C12A7328-F81F-11D2-BA4B-00A0C93EC93B')
image.partition(change_name='2:system-boot')
image.copy_blob(self.boot_img,
bs='1MB', seek=4, count=64, conv='notrunc')
# Create main snappy writable partition
image.partition(new='3:72MiB:+3646MiB')
image.partition(typecode='3:0FC63DAF-8483-4772-8E79-3D69D8477DE4')
image.partition(change_name='3:writable')
image.copy_blob(self.root_img,
bs='1MiB', seek=72, count=3646, conv='notrunc')
self._next.append(self.finish)
def _populate_one_volume(self, name, volume):
for partnum, part in enumerate(volume.structures):
part_img = volume.part_images[partnum]
part_dir = os.path.join(volume.basedir, 'part{}'.format(partnum))
if part.role is StructureRole.system_data:
# The root partition needs to be ext4, which may or may not be
# populated at creation time, depending on the version of
# e2fsprogs.
mkfs_ext4(part_img,
self.rootfs,
self.args.cmd,
part.filesystem_label,
preserve_ownership=True)
elif part.filesystem is FileSystemType.none:
image = Image(part_img, part.size)
offset = 0
for content in part.content:
src = os.path.join(self.unpackdir, 'gadget', content.image)
file_size = os.path.getsize(src)
assert content.size is None or content.size >= file_size, (
'Spec size {} < actual size {} of: {}'.format(
content.size, file_size, content.image))
if content.size is not None:
file_size = content.size
# TODO: We need to check for overlapping images.
if content.offset is not None:
offset = content.offset
end = offset + file_size
if end > part.size:
if part.name is None:
if part.role is None:
whats_wrong = part.type
else:
whats_wrong = part.role.value
else:
whats_wrong = part.name
part_path = 'volumes:<{}>:structure:<{}>'.format(
name, whats_wrong)
self.exitcode = 1
raise DoesNotFit(partnum, part_path, end - part.size)
image.copy_blob(src, bs=1, seek=offset, conv='notrunc')
offset += file_size
elif part.filesystem is FileSystemType.vfat:
sourcefiles = SPACE.join(
os.path.join(part_dir, filename)
for filename in os.listdir(part_dir))
env = dict(MTOOLS_SKIP_CHECK='1')
env.update(os.environ)
run('mcopy -s -i {} {} ::'.format(part_img, sourcefiles),
env=env)
elif part.filesystem is FileSystemType.ext4:
mkfs_ext4(part_img, part_dir, self.args.cmd,
part.filesystem_label)
else:
raise AssertionError('Invalid part filesystem type: {}'.format(
part.filesystem))
def _populate_one_volume(self, name, volume):
for partnum, part in enumerate(volume.structures):
part_img = volume.part_images[partnum]
part_dir = os.path.join(volume.basedir, 'part{}'.format(partnum))
if part.role is StructureRole.system_data:
# The root partition needs to be ext4, which may or may not be
# populated at creation time, depending on the version of
# e2fsprogs.
mkfs_ext4(part_img, self.rootfs, self.args.cmd,
part.filesystem_label, preserve_ownership=True)
elif part.filesystem is FileSystemType.none:
image = Image(part_img, part.size)
offset = 0
for content in part.content:
src = os.path.join(self.unpackdir, 'gadget', content.image)
file_size = os.path.getsize(src)
assert content.size is None or content.size >= file_size, (
'Spec size {} < actual size {} of: {}'.format(
content.size, file_size, content.image))
if content.size is not None:
file_size = content.size
# TODO: We need to check for overlapping images.
if content.offset is not None:
offset = content.offset
end = offset + file_size
if end > part.size:
if part.name is None:
if part.role is None:
whats_wrong = part.type
else:
whats_wrong = part.role.value
else:
whats_wrong = part.name
part_path = 'volumes:<{}>:structure:<{}>'.format(
name, whats_wrong)
self.exitcode = 1
raise DoesNotFit(partnum, part_path, end - part.size)
image.copy_blob(src, bs=1, seek=offset, conv='notrunc')
offset += file_size
elif part.filesystem is FileSystemType.vfat:
sourcefiles = SPACE.join(
os.path.join(part_dir, filename)
for filename in os.listdir(part_dir)
)
env = dict(MTOOLS_SKIP_CHECK='1')
env.update(os.environ)
run('mcopy -s -i {} {} ::'.format(part_img, sourcefiles),
env=env)
elif part.filesystem is FileSystemType.ext4:
mkfs_ext4(part_img, part_dir, self.args.cmd,
part.filesystem_label)
else:
raise AssertionError('Invalid part filesystem type: {}'.format(
part.filesystem))
def test_copy_blob_with_seek(self):
# dd if=blobs/img.bios-boot of=img bs=1MiB seek=4 count=1 conv=notrunc
blob_file = os.path.join(self.tmpdir, 'img.bios-boot')
with open(blob_file, 'wb') as fp:
fp.write(b'x' * 100)
image = Image(self.img, MiB(2))
image.copy_blob(blob_file, bs=773, seek=4, count=1, conv='notrunc')
# The seek=4 skipped 4 blocks of 773 bytes.
with open(image.path, 'rb') as fp:
self.assertEqual(fp.read(3092), b'\0' * 3092)
self.assertEqual(fp.read(100), b'x' * 100)
self.assertEqual(fp.read(25), b'\0' * 25)
def test_copy_blob_with_seek(self):
# dd if=blobs/img.bios-boot of=img bs=1MiB seek=4 count=1 conv=notrunc
blob_file = os.path.join(self.tmpdir, 'img.bios-boot')
with open(blob_file, 'wb') as fp:
fp.write(b'x' * 100)
image = Image(self.img, MiB(2))
image.copy_blob(blob_file, bs=773, seek=4, count=1, conv='notrunc')
# The seek=4 skipped 4 blocks of 773 bytes.
with open(image.path, 'rb') as fp:
self.assertEqual(fp.read(3092), b'\0' * 3092)
self.assertEqual(fp.read(100), b'x' * 100)
self.assertEqual(fp.read(25), b'\0' * 25)
def _make_one_disk(self, imgfile, name, volume):
part_id = 1
# Create the image object for the selected volume schema
image = Image(imgfile, volume.image_size, volume.schema)
offset_writes = []
part_offsets = {}
# We first create all the needed partitions.
# For regular core16 and core18 builds, this means creating all of the
# defined partitions. For core20 (the so called 'seeded images'), we
# only create all the role-less partitions and mbr + system-seed.
# The rest is created dynamically by snapd on first boot.
for part in volume.structures:
if part.name is not None:
part_offsets[part.name] = part.offset
if part.offset_write is not None:
offset_writes.append((part.offset, part.offset_write))
if (part.role is StructureRole.mbr or part.type == 'bare' or
self._should_skip_partition(part)):
continue
activate = False
if (volume.schema is VolumeSchema.mbr and
part.role is StructureRole.system_boot):
activate = True
elif (volume.schema is VolumeSchema.gpt and
part.role is StructureRole.system_data and
part.name is None):
part.name = 'writable'
image.partition(part.offset, part.size, part.name, activate)
# Now since we're done, we need to do a second pass to copy the data
# and set all the partition types. This needs to be done like this as
# libparted's commit() operation resets type GUIDs to defaults and
# clobbers things like hybrid MBR partitions.
part_id = 1
for i, part in enumerate(volume.structures):
if self._should_skip_partition(part):
continue
image.copy_blob(volume.part_images[i],
bs=image.sector_size,
seek=part.offset // image.sector_size,
count=ceil(part.size / image.sector_size),
conv='notrunc')
if part.role is StructureRole.mbr or part.type == 'bare':
continue
image.set_parition_type(part_id, part.type)
part_id += 1
for value, dest in offset_writes:
# Decipher non-numeric offset_write values.
if isinstance(dest, tuple):
dest = part_offsets[dest[0]] + dest[1]
image.write_value_at_offset(value // image.sector_size, dest)
def _make_one_disk(self, imgfile, name, volume):
part_id = 1
# Create the image object for the selected volume schema
image = Image(imgfile, volume.image_size, volume.schema)
offset_writes = []
part_offsets = {}
# We first create all the partitions.
for part in volume.structures:
if part.name is not None:
part_offsets[part.name] = part.offset
if part.offset_write is not None:
offset_writes.append((part.offset, part.offset_write))
if part.role is StructureRole.mbr or part.type == 'bare':
continue
activate = False
if (volume.schema is VolumeSchema.mbr and
part.role is StructureRole.system_boot):
activate = True
elif (volume.schema is VolumeSchema.gpt and
part.role is StructureRole.system_data and
part.name is None):
part.name = 'writable'
image.partition(part.offset, part.size, part.name, activate)
# Now since we're done, we need to do a second pass to copy the data
# and set all the partition types. This needs to be done like this as
# libparted's commit() operation resets type GUIDs to defaults and
# clobbers things like hybrid MBR partitions.
part_id = 1
for i, part in enumerate(volume.structures):
image.copy_blob(volume.part_images[i],
bs=image.sector_size,
seek=part.offset // image.sector_size,
count=ceil(part.size / image.sector_size),
conv='notrunc')
if part.role is StructureRole.mbr or part.type == 'bare':
continue
image.set_parition_type(part_id, part.type)
part_id += 1
for value, dest in offset_writes:
# Decipher non-numeric offset_write values.
if isinstance(dest, tuple):
dest = part_offsets[dest[0]] + dest[1]
image.write_value_at_offset(value // image.sector_size, dest)
def populate_filesystems(self):
volumes = self.gadget.volumes.values()
assert len(volumes) == 1, 'For now, only one volume is allowed'
volume = list(volumes)[0]
for partnum, part in enumerate(volume.structures):
part_img = self.boot_images[partnum]
part_dir = os.path.join(self.workdir, 'part{}'.format(partnum))
if part.filesystem is FileSystemType.none: # pragma: nocover
image = Image(part_img, part.size)
offset = 0
for file in part.content:
src = os.path.join(self.unpackdir, 'gadget', file.image)
file_size = os.path.getsize(src)
if file.size is not None and file.size < file_size:
raise AssertionError('Size {} < size of {}'.format(
file.size, file.image))
if file.size is not None:
file_size = file.size
# XXX: We need to check for overlapping images.
if file.offset is not None:
offset = file.offset
# XXX: We must check offset+size vs. the target image.
image.copy_blob(src, bs=1, seek=offset, conv='notrunc')
offset += file_size
elif part.filesystem is FileSystemType.vfat: # pragma: nobranch
sourcefiles = SPACE.join(
os.path.join(part_dir, filename)
for filename in os.listdir(part_dir))
env = dict(MTOOLS_SKIP_CHECK='1')
env.update(os.environ)
run('mcopy -s -i {} {} ::'.format(part_img, sourcefiles),
env=env)
elif part.filesystem is FileSystemType.ext4: # pragma: nocover
_mkfs_ext4(self.part_img, part_dir, part.filesystem_label)
# The root partition needs to be ext4, which may or may not be
# populated at creation time, depending on the version of e2fsprogs.
_mkfs_ext4(self.root_img, self.rootfs)
self._next.append(self.make_disk)
def _populate_one_volume(self, name, volume):
# For the LK bootloader we need to copy boot.img and snapbootsel.bin to
# the gadget folder so they can be used as partition content. The first
# one comes from the kernel snap, while the second one is modified by
# 'snap prepare-image' to set the right core and kernel for the kernel
# command line.
if volume.bootloader is BootLoader.lk:
boot = os.path.join(self.unpackdir, 'image', 'boot', 'lk')
gadget = os.path.join(self.unpackdir, 'gadget')
if os.path.isdir(boot):
os.makedirs(gadget, exist_ok=True)
for filename in os.listdir(boot):
src = os.path.join(boot, filename)
dst = os.path.join(gadget, filename)
shutil.copy(src, dst)
for partnum, part in enumerate(volume.structures):
part_img = volume.part_images[partnum]
# In seeded images, the system-seed partition is basically the
# rootfs partition - at least from the ubuntu-image POV.
if part.role is StructureRole.system_seed:
part_dir = self.rootfs
else:
part_dir = os.path.join(volume.basedir,
'part{}'.format(partnum))
if part.role is StructureRole.system_data:
# The root partition needs to be ext4, which may or may not be
# populated at creation time, depending on the version of
# e2fsprogs.
mkfs_ext4(part_img,
self.rootfs,
self.args.cmd,
part.filesystem_label,
preserve_ownership=True)
elif part.filesystem is FileSystemType.none:
image = Image(part_img, part.size)
offset = 0
for content in part.content:
src = os.path.join(self.unpackdir, 'gadget', content.image)
file_size = os.path.getsize(src)
assert content.size is None or content.size >= file_size, (
'Spec size {} < actual size {} of: {}'.format(
content.size, file_size, content.image))
if content.size is not None:
file_size = content.size
# TODO: We need to check for overlapping images.
if content.offset is not None:
offset = content.offset
end = offset + file_size
if end > part.size:
if part.name is None:
if part.role is None:
whats_wrong = part.type
else:
whats_wrong = part.role.value
else:
whats_wrong = part.name
part_path = 'volumes:<{}>:structure:<{}>'.format(
name, whats_wrong)
self.exitcode = 1
raise DoesNotFit(partnum, part_path, end - part.size)
image.copy_blob(src, bs=1, seek=offset, conv='notrunc')
offset += file_size
elif part.filesystem is FileSystemType.vfat:
sourcefiles = SPACE.join(
os.path.join(part_dir, filename)
for filename in os.listdir(part_dir))
env = dict(MTOOLS_SKIP_CHECK='1')
env.update(os.environ)
run('mcopy -s -i {} {} ::'.format(part_img, sourcefiles),
env=env)
elif part.filesystem is FileSystemType.ext4:
mkfs_ext4(part_img, part_dir, self.args.cmd,
part.filesystem_label)
else:
raise AssertionError('Invalid part filesystem type: {}'.format(
part.filesystem))
#!/usr/bin/python3
from ubuntu_image.image import Diagnostics, GiB, Image
# Create empty image of a fixed size
# TODO: compute rough good size after seeing all the required snaps.
image = Image('img', GiB(4))
# Install GRUB to MBR
# TODO: this has to be represented in the image.yaml
# NOTE: the boot.img has to be a part of the gadget snap itself
# FIXME: embed a pointer to 2nd stage in bios-boot partition
image.copy_blob('blogs/img.mbr', bs=446, count=1, conv='notrunc')
# Create BIOS boot partition
#
# The partition is 1MiB in size, as recommended by various partitioning guides.
# The actual required size is much, much smaller.
#
# https://www.gnu.org/software/grub/manual/html_node/BIOS-installation.html#BIOS-installation
image.partition(new='1:4MiB:+1MiB')
image.partition(typecode='1:21686148-6449-6E6F-744E-656564454649')
image.partition(change_name='1:grub')
image.copy_blob('blobs/img.bios-boot',
bs='1MiB', seek=4, count=1, conv='notrunc')
# Create EFI system partition
#
# TODO: switch to 512MiB as recommended by the standard
image.partition(new='2:5MiB:+64MiB')
#!/usr/bin/python3
from ubuntu_image.image import Diagnostics, GiB, Image
# Create empty image of a fixed size
# TODO: compute rough good size after seeing all the required snaps.
image = Image('img', GiB(4))
# Install GRUB to MBR
# TODO: this has to be represented in the image.yaml
# NOTE: the boot.img has to be a part of the gadget snap itself
# FIXME: embed a pointer to 2nd stage in bios-boot partition
image.copy_blob('blogs/img.mbr', bs=446, count=1, conv='notrunc')
# Create BIOS boot partition
#
# The partition is 1MiB in size, as recommended by various partitioning guides.
# The actual required size is much, much smaller.
#
# https://www.gnu.org/software/grub/manual/html_node/BIOS-installation.html#BIOS-installation
image.partition(new='1:4MiB:+1MiB')
image.partition(typecode='1:21686148-6449-6E6F-744E-656564454649')
image.partition(change_name='1:grub')
image.copy_blob('blobs/img.bios-boot',
bs='1MiB',
seek=4,
count=1,
conv='notrunc')
# Create EFI system partition
