def _TestDiskBundleHelper(self, partition_start, partition_end, fs_uuid):
disk_path = self._SetupMbrDisk(partition_start, partition_end, fs_uuid)
with utils.LoadDiskImage(disk_path) as devices:
# Get the path to do the disk.
# devices will have something which is like /dev/mapper/loop0p1
# We need to get loop0 out of it.
disk_loop_back_path = '/dev/' + devices[0].split('/')[3][:-2]
# Create a symlinks to the disk and loopback paths
# This is required because of the code where we assume first
# partition is device path appended by 1. Will remove it once we
# update that part of the code.
symlink_disk = os.path.join(self.tmp_root, 'disk')
symlink_partition = self.tmp_root + '/disk1'
utils.RunCommand(['ln', '-s', disk_loop_back_path, symlink_disk])
utils.RunCommand(['ln', '-s', devices[0], symlink_partition])
# Bundle up
self._bundle.AddDisk(symlink_disk)
self._bundle.AddSource(self.tmp_path)
self._bundle.Verify()
(_, _) = self._bundle.Bundleup()
self._VerifyImageHas(self._tar_path, [
'lost+found', 'test1', 'test2', 'dir1/', '/dir1/dir11/',
'/dir1/sl1', '/dir1/hl2', 'dir2/', '/dir2/dir1', '/dir2/sl2',
'/dir2/hl1'
])
self._VerifyNumberOfHardLinksInRawDisk(self._tar_path, 'test1', 2)
self._VerifyNumberOfHardLinksInRawDisk(self._tar_path, 'test2', 2)
self._VerifyDiskSize(self._tar_path, self._fs_size)
self._VerifyNonPartitionContents(self._tar_path, disk_path,
partition_start)
self._VerifyFilesystemUUID(self._tar_path, fs_uuid)
def _VerifyFilesystemUUID(self, tar, expected_uuid):
"""Verifies UUID of the first partition on disk matches the value."""
tmp_dir = tempfile.mkdtemp(dir=self.tmp_root)
tar_cmd = ['tar', '-xzf', tar, '-C', tmp_dir]
self.assertEqual(subprocess.call(tar_cmd), 0)
created_disk_path = os.path.join(tmp_dir, 'disk.raw')
with utils.LoadDiskImage(created_disk_path) as devices:
self.assertEqual(1, len(devices))
self.assertEqual(expected_uuid, utils.GetUUID(devices[0]))
def _VerifyNumberOfHardLinksInRawDisk(self, tar, filename, count):
"""Tests if a file on raw disk has a specified number of hard links."""
tmp_dir = tempfile.mkdtemp(dir=self.tmp_root)
tar_cmd = ['tar', '-xzf', tar, '-C', tmp_dir]
self.assertEqual(subprocess.call(tar_cmd), 0)
disk_path = os.path.join(tmp_dir, 'disk.raw')
with utils.LoadDiskImage(disk_path) as devices:
self.assertEqual(len(devices), 1)
mnt_dir = tempfile.mkdtemp(dir=self.tmp_root)
with utils.MountFileSystem(devices[0], mnt_dir):
self.assertEqual(
os.stat(os.path.join(mnt_dir, filename)).st_nlink, count)
def _InitializeDiskFileFromDevice(self, file_path):
"""Initializes disk file from the device specified in self._disk.
It preserves whatever may be there on the device prior to the start of the
first partition.
At the moment this method supports devices with a single partition only.
Args:
file_path: The path where the disk file should be created.
Returns:
A tuple with partition_start, uuid. partition_start is the location
where the first partition on the disk starts and uuid is the filesystem
UUID to use for the first partition.
Raises:
RawDiskError: If there are more than one partition on the disk device.
"""
# Find the disk size
disk_size = utils.GetDiskSize(self._disk)
logging.debug('Size of disk is %s', disk_size)
# Make the disk file big enough to hold the disk
self._ResizeFile(file_path, disk_size)
# Find the location where the first partition starts
partition_start = utils.GetPartitionStart(self._disk, 1)
logging.debug('First partition starts at %s', partition_start)
# Copy all the bytes as is from the start of the disk to the start of
# first partition
utils.CopyBytes(self._disk, file_path, partition_start)
# Verify there is only 1 partition on the disk
with utils.LoadDiskImage(file_path) as devices:
# For now we only support disks with a single partition.
if len(devices) == 0:
raise RawDiskError(
'Device %s should be a disk not a partition.' % self._disk)
elif len(devices) != 1:
raise RawDiskError(
'Device %s has more than 1 partition. Only devices '
'with a single partition are supported.' % self._disk)
# Remove the first partition from the file we are creating. We will
# recreate a partition that will fit inside _fs_size later.
utils.RemovePartition(file_path, 1)
# Resize the disk.raw file down to self._fs_size
# We do this after removing the first partition to ensure that an
# existing partition doesn't fall outside the boundary of the disk device.
self._ResizeFile(file_path, self._fs_size)
# Get UUID of the first partition on the disk
# TODO(user): This is very hacky and relies on the disk path being
# similar to /dev/sda etc which is bad. Need to fix it.
uuid = utils.GetUUID(self._disk + '1')
return partition_start, uuid
def _VerifyFileInRawDiskEndsWith(self, tar, filename, text):
"""Tests if a file on raw disk contains ends with a specified text."""
tmp_dir = tempfile.mkdtemp(dir=self.tmp_root)
tar_cmd = ['tar', '-xzf', tar, '-C', tmp_dir]
self.assertEqual(subprocess.call(tar_cmd), 0)
disk_path = os.path.join(tmp_dir, 'disk.raw')
with utils.LoadDiskImage(disk_path) as devices:
self.assertEqual(len(devices), 1)
mnt_dir = tempfile.mkdtemp(dir=self.tmp_root)
with utils.MountFileSystem(devices[0], mnt_dir):
f = open(os.path.join(mnt_dir, filename), 'r')
file_content = f.read()
f.close()
self.assertTrue(file_content.endswith(text))
def _VerifyImageHas(self, tar, expected):
"""Tests if raw disk contains an expected list of files/directories."""
tmp_dir = tempfile.mkdtemp(dir=self.tmp_root)
tar_cmd = ['tar', '-xzf', tar, '-C', tmp_dir]
self.assertEqual(subprocess.call(tar_cmd), 0)
disk_path = os.path.join(tmp_dir, 'disk.raw')
with utils.LoadDiskImage(disk_path) as devices:
self.assertEqual(len(devices), 1)
mnt_dir = tempfile.mkdtemp(dir=self.tmp_root)
with utils.MountFileSystem(devices[0], mnt_dir):
found = []
for root, dirs, files in os.walk(mnt_dir):
root = root.replace(mnt_dir, '')
for f in files:
found.append(os.path.join(root, f))
for d in dirs:
found.append(os.path.join(root, d))
self._AssertListEqual(expected, found)
def _SetupMbrDisk(self, partition_start, partition_end, fs_uuid):
"""Creates a disk with a fake MBR.
Args:
partition_start: The byte offset where the partition starts.
partition_end: The byte offset where the partition ends.
fs_uuid: The UUID of the filesystem to create on the partition.
Returns:
The path where the disk is located.
"""
# Create the disk file with the size specified.
disk_path = os.path.join(self.tmp_root, 'mbrdisk.raw')
disk_size = partition_end + FsRawDiskTest._MEGABYTE
with open(disk_path, 'wb') as disk_file:
disk_file.truncate(disk_size)
# Create a partition table
utils.MakePartitionTable(disk_path)
# Create the partition
utils.MakePartition(disk_path, 'primary', 'ext2', partition_start,
partition_end)
# Create the file system
with utils.LoadDiskImage(disk_path) as devices:
utils.MakeFileSystem(devices[0], 'ext4', fs_uuid)
# Write some data after the MBR but before the first partition
with open(disk_path, 'r+b') as disk_file:
# Seek to last two bytes of first sector
disk_file.seek(510)
# Write MBR signature
disk_file.write(chr(0x55))
disk_file.write(chr(0xAA))
# Write random data on the disk till the point first partition starts
for _ in range(partition_start - 512):
# Write a byte
disk_file.write(chr(random.randint(0, 127)))
return disk_path
def Bundleup(self):
"""Creates a raw disk copy of OS image and bundles it into gzipped tar.
Returns:
A size of a generated raw disk and the SHA1 digest of the the tar archive.
Raises:
RawDiskError: If number of partitions in a created image doesn't match
expected count.
"""
# Create sparse file with specified size
disk_file_path = os.path.join(self._scratch_dir, self._disk_file_name)
# with open(disk_file_path, 'wb') as _:
# pass
self._excludes.append(exclude_spec.ExcludeSpec(disk_file_path))
logging.info('Initializing disk file')
partition_start = None
uuid = None
if self._disk:
# If a disk device has been provided then preserve whatever is there on
# the disk before the first partition in case there is an MBR present.
partition_start, uuid = self._InitializeDiskFileFromDevice(disk_file_path)
else:
#feoff: our code goes this way
# User didn't specify a disk device. Initialize a device with a simple
# partition table.
logging.info("Setting partiton size: " + str(self._fs_size) )
self._ResizeFile(disk_file_path, self._fs_size)
# User didn't specify a disk to copy. Create a new partition table
utils.MakePartitionTable(disk_file_path)
# Pass 1MB as start to avoid 'Warning: The resulting partition is not
# properly aligned for best performance.' from parted.
partition_start = 1024 * 1024
# Create a new partition starting at partition_start of size
# self._fs_size - partition_start.
#feoff: it's not just a log ensure the size changed. we ensure the size is changed and OS updated info on it. needed in FUSE scenario
utils.MakePartition(disk_file_path, 'primary', 'ext2', partition_start,
self._fs_size-512) # feoff: THE LAST PARM IS THE LAST SECTOR! , not size!
logging.info("Preparing file disk size " + str(os.path.getsize(disk_file_path)) + " bytes")
with utils.LoadDiskImage(disk_file_path) as devices:
# For now we only support disks with a single partition.
if len(devices) != 1:
raise RawDiskError(devices)
# List contents of /dev/mapper to help with debugging. Contents will
# be listed in debug log only
utils.RunCommand(['ls', '/dev/mapper'])
logging.info('Making filesystem')
uuid = utils.MakeFileSystem(devices[0], self._fs_type, uuid)
with utils.LoadDiskImage(disk_file_path) as devices:
if uuid is None:
raise Exception('Could not get uuid from MakeFileSystem')
mount_point = tempfile.mkdtemp(dir=self._scratch_dir)
with utils.MountFileSystem(devices[0], mount_point, self._fs_type):
logging.info('Copying contents')
#feoff: temporary disable selinux to ensure rsync works fine, see https://github.com/GoogleCloudPlatform/compute-image-packages/issues/132
selinux_state = self._setSELinux("0")
self._CopySourceFiles(mount_point)
self._CopyPlatformSpecialFiles(mount_point)
self._ProcessOverwriteList(mount_point)
self._CleanupNetwork(mount_point)
self._UpdateFstab(mount_point, uuid)
# feoff: set dhcp to eth0
self._SetDhcp(mount_point)
# feoff: grub
# should move add_grub to the parm
add_grub = True
if add_grub:
from gcimagebundlelib import grub
grub.InstallGrub(mount_point, devices[0])
self._setSELinux(selinux_state)
tar_entries = []
manifest_file_path = os.path.join(self._scratch_dir, 'manifest.json')
manifest_created = self._manifest.CreateIfNeeded(manifest_file_path)
if manifest_created:
tar_entries.append(manifest_file_path)
tar_entries.append(disk_file_path)
#TODO(feoff): make it parametrizable - if to start tar or not
# logging.info('Creating tar.gz archive')
utils.TarAndGzipFile(tar_entries,
self._output_tarfile)
# Removed the deletion of file
for tar_entry in tar_entries:
if not self._output_tarfile == tar_entry:
os.remove(tar_entry)
# TODO(user): It would be better to compute tar.gz file hash during
# archiving.
h = hashlib.sha1()
with open(self._output_tarfile, 'rb') as tar_file:
for chunk in iter(lambda: tar_file.read(8192), ''):
h.update(chunk)
return (self._fs_size, h.hexdigest())
def Bundleup(self):
"""Creates a raw disk copy of OS image and bundles it into gzipped tar.
Returns:
A size of a generated raw disk and the SHA1 digest of the the tar archive.
Raises:
RawDiskError: If number of partitions in a created image doesn't match
expected count.
"""
# Create sparse file with specified size
disk_file_path = os.path.join(self._scratch_dir, 'disk.raw')
with open(disk_file_path, 'wb') as _:
pass
self._excludes.append(exclude_spec.ExcludeSpec(disk_file_path))
logging.info('Initializing disk file')
partition_start = None
uuid = None
if self._disk:
# If a disk device has been provided then preserve whatever is there on
# the disk before the first partition in case there is an MBR present.
partition_start, uuid = self._InitializeDiskFileFromDevice(
disk_file_path)
else:
# User didn't specify a disk device. Initialize a device with a simple
# partition table.
self._ResizeFile(disk_file_path, self._fs_size)
# User didn't specify a disk to copy. Create a new partition table
utils.MakePartitionTable(disk_file_path)
# Pass 1MB as start to avoid 'Warning: The resulting partition is not
# properly aligned for best performance.' from parted.
partition_start = 1024 * 1024
# Create a new partition starting at partition_start of size
# self._fs_size - partition_start
utils.MakePartition(disk_file_path, 'primary', 'ext2', partition_start,
self._fs_size - partition_start)
with utils.LoadDiskImage(disk_file_path) as devices:
# For now we only support disks with a single partition.
if len(devices) != 1:
raise RawDiskError(devices)
# List contents of /dev/mapper to help with debugging. Contents will
# be listed in debug log only
utils.RunCommand(['ls', '/dev/mapper'])
logging.info('Making filesystem')
uuid = utils.MakeFileSystem(devices[0], self._fs_type, uuid)
with utils.LoadDiskImage(disk_file_path) as devices:
if uuid is None:
raise Exception('Could not get uuid from MakeFileSystem')
mount_point = tempfile.mkdtemp(dir=self._scratch_dir)
with utils.MountFileSystem(devices[0], mount_point):
logging.info('Copying contents')
self._CopySourceFiles(mount_point)
self._CopyPlatformSpecialFiles(mount_point)
self._ProcessOverwriteList(mount_point)
self._CleanupNetwork(mount_point)
self._UpdateFstab(mount_point, uuid)
tar_entries = []
manifest_file_path = os.path.join(self._scratch_dir, 'manifest.json')
manifest_created = self._manifest.CreateIfNeeded(manifest_file_path)
if manifest_created:
tar_entries.append(manifest_file_path)
tar_entries.append(disk_file_path)
logging.info('Creating tar.gz archive')
utils.TarAndGzipFile(tar_entries, self._output_tarfile)
for tar_entry in tar_entries:
os.remove(tar_entry)
# TODO(user): It would be better to compute tar.gz file hash during
# archiving.
h = hashlib.sha1()
with open(self._output_tarfile, 'rb') as tar_file:
for chunk in iter(lambda: tar_file.read(8192), ''):
h.update(chunk)
return (self._fs_size, h.hexdigest())
