def _TestDiskBundleHelper(self, partition_start, partition_end, fs_uuid):
disk_path = self._SetupMbrDisk(partition_start, partition_end, fs_uuid)
with utils.LoadDiskImage(disk_path) as devices:
# Get the path to do the disk.
# devices will have something which is like /dev/mapper/loop0p1
# We need to get loop0 out of it.
disk_loop_back_path = '/dev/' + devices[0].split('/')[3][:-2]
# Create a symlinks to the disk and loopback paths
# This is required because of the code where we assume first
# partition is device path appended by 1. Will remove it once we
# update that part of the code.
symlink_disk = os.path.join(self.tmp_root, 'disk')
symlink_partition = self.tmp_root + '/disk1'
utils.RunCommand(['ln', '-s', disk_loop_back_path, symlink_disk])
utils.RunCommand(['ln', '-s', devices[0], symlink_partition])
# Bundle up
self._bundle.AddDisk(symlink_disk)
self._bundle.AddSource(self.tmp_path)
self._bundle.Verify()
(_, _) = self._bundle.Bundleup()
self._VerifyImageHas(self._tar_path, [
'lost+found', 'test1', 'test2', 'dir1/', '/dir1/dir11/',
'/dir1/sl1', '/dir1/hl2', 'dir2/', '/dir2/dir1', '/dir2/sl2',
'/dir2/hl1'
])
self._VerifyNumberOfHardLinksInRawDisk(self._tar_path, 'test1', 2)
self._VerifyNumberOfHardLinksInRawDisk(self._tar_path, 'test2', 2)
self._VerifyDiskSize(self._tar_path, self._fs_size)
self._VerifyNonPartitionContents(self._tar_path, disk_path,
partition_start)
self._VerifyFilesystemUUID(self._tar_path, fs_uuid)
def testDiskBundle(self):
"""Tests bundle command when a disk is specified.
Creates a 20Gb source disk to start with and verifies that creating
a 10MB file off it works.
"""
# Create a 20GB disk with first partition starting at 1MB
self._TestDiskBundleHelper(FsRawDiskTest._MEGABYTE,
FsRawDiskTest._GIGABYTE * 20,
utils.RunCommand(['uuidgen']).strip())
def testDiskBundlePartitionAt2MB(self):
"""Tests bundle command when a disk is specified.
Creates the first partition at 2MB and verifies all data prior to that is
copied.
"""
# Create a 20GB disk with first partition starting at 2MB
self._TestDiskBundleHelper(FsRawDiskTest._MEGABYTE * 2,
FsRawDiskTest._GIGABYTE * 20,
utils.RunCommand(['uuidgen']).strip())
def main():
parser = SetupArgsParser()
(options, _) = parser.parse_args()
if options.display_version:
PrintVersionInfo()
return 0
EnsureSuperUser()
VerifyArgs(parser, options)
scratch_dir = tempfile.mkdtemp(dir=options.output_directory)
SetupLogging(options, scratch_dir)
logging.warn(
'============================================================\n'
'Warning: gcimagebundle is deprecated. See\n'
'https://cloud.google.com/compute/docs/creating-custom-image'
'#export_an_image_to_google_cloud_storage\n'
'for updated instructions.\n'
'============================================================')
try:
guest_platform = platform_factory.PlatformFactory(
options.root_directory).GetPlatform()
except platform_factory.UnknownPlatformException:
logging.critical(
'Platform is not supported.'
' Platform rules can be added to platform_factory.py.')
return -1
temp_file_name = tempfile.mktemp(dir=scratch_dir, suffix='.tar.gz')
file_system = GetTargetFilesystem(options, guest_platform)
logging.info('File System: %s', file_system)
logging.info('Disk Size: %s bytes', options.fs_size)
bundle = block_disk.RootFsRaw(options.fs_size, file_system,
options.skip_disk_space_check)
bundle.SetTarfile(temp_file_name)
if options.disk:
readlink_command = ['readlink', '-f', options.disk]
final_path = utils.RunCommand(readlink_command).strip()
logging.info('Resolved %s to %s', options.disk, final_path)
bundle.AddDisk(final_path)
# TODO(user): Find the location where the first partition of the disk
# is mounted and add it as the source instead of relying on the source
# param flag
bundle.AddSource(options.root_directory)
bundle.SetKey(options.key)
bundle.SetScratchDirectory(scratch_dir)
# Merge platform specific exclude list, mounts points
# and user specified excludes
excludes = guest_platform.GetExcludeList()
if options.excludes:
excludes.extend(
[exclude_spec.ExcludeSpec(x) for x in options.excludes.split(',')])
logging.info('exclude list: %s', ' '.join([x.GetSpec() for x in excludes]))
bundle.AppendExcludes(excludes)
if not options.include_mounts:
mount_points = utils.GetMounts(options.root_directory)
logging.info('ignoring mounts %s', ' '.join(mount_points))
bundle.AppendExcludes([
exclude_spec.ExcludeSpec(x, preserve_dir=True)
for x in utils.GetMounts(options.root_directory)
])
bundle.SetPlatform(guest_platform)
# Verify that bundle attributes are correct and create tar bundle.
bundle.Verify()
(fs_size, digest) = bundle.Bundleup()
if not digest:
logging.critical('Could not get digest for the bundle.'
' The bundle may not be created correctly')
return -1
if fs_size > options.fs_size:
logging.critical('Size of tar %d exceeds the file system size %d.',
fs_size, options.fs_size)
return -1
if options.output_file_name:
output_file = os.path.join(options.output_directory,
options.output_file_name)
else:
output_file = os.path.join(options.output_directory,
'%s.image.tar.gz' % digest)
os.rename(temp_file_name, output_file)
logging.info('Created tar.gz file at %s' % output_file)
if options.bucket:
bucket = options.bucket
if bucket.startswith('gs://'):
output_bucket = '%s/%s' % (bucket, os.path.basename(output_file))
else:
output_bucket = 'gs://%s/%s' % (bucket,
os.path.basename(output_file))
# /usr/local/bin not in redhat root PATH by default
if '/usr/local/bin' not in os.environ['PATH']:
os.environ['PATH'] += ':/usr/local/bin'
# TODO: Consider using boto library directly.
cmd = ['gsutil', 'cp', output_file, output_bucket]
retcode = subprocess.call(cmd)
if retcode != 0:
logging.critical(
'Failed to copy image to bucket. '
'gsutil returned %d. To retry, run the command: %s', retcode,
' '.join(cmd))
return -1
logging.info('Uploaded image to %s', output_bucket)
# If we've uploaded, then we can remove the local file.
os.remove(output_file)
if options.cleanup:
shutil.rmtree(scratch_dir)
def Bundleup(self):
"""Creates a raw disk copy of OS image and bundles it into gzipped tar.
Returns:
A size of a generated raw disk and the SHA1 digest of the the tar archive.
Raises:
RawDiskError: If number of partitions in a created image doesn't match
expected count.
"""
# Create sparse file with specified size
disk_file_path = os.path.join(self._scratch_dir, self._disk_file_name)
# with open(disk_file_path, 'wb') as _:
# pass
self._excludes.append(exclude_spec.ExcludeSpec(disk_file_path))
logging.info('Initializing disk file')
partition_start = None
uuid = None
if self._disk:
# If a disk device has been provided then preserve whatever is there on
# the disk before the first partition in case there is an MBR present.
partition_start, uuid = self._InitializeDiskFileFromDevice(disk_file_path)
else:
#feoff: our code goes this way
# User didn't specify a disk device. Initialize a device with a simple
# partition table.
logging.info("Setting partiton size: " + str(self._fs_size) )
self._ResizeFile(disk_file_path, self._fs_size)
# User didn't specify a disk to copy. Create a new partition table
utils.MakePartitionTable(disk_file_path)
# Pass 1MB as start to avoid 'Warning: The resulting partition is not
# properly aligned for best performance.' from parted.
partition_start = 1024 * 1024
# Create a new partition starting at partition_start of size
# self._fs_size - partition_start.
#feoff: it's not just a log ensure the size changed. we ensure the size is changed and OS updated info on it. needed in FUSE scenario
utils.MakePartition(disk_file_path, 'primary', 'ext2', partition_start,
self._fs_size-512) # feoff: THE LAST PARM IS THE LAST SECTOR! , not size!
logging.info("Preparing file disk size " + str(os.path.getsize(disk_file_path)) + " bytes")
with utils.LoadDiskImage(disk_file_path) as devices:
# For now we only support disks with a single partition.
if len(devices) != 1:
raise RawDiskError(devices)
# List contents of /dev/mapper to help with debugging. Contents will
# be listed in debug log only
utils.RunCommand(['ls', '/dev/mapper'])
logging.info('Making filesystem')
uuid = utils.MakeFileSystem(devices[0], self._fs_type, uuid)
with utils.LoadDiskImage(disk_file_path) as devices:
if uuid is None:
raise Exception('Could not get uuid from MakeFileSystem')
mount_point = tempfile.mkdtemp(dir=self._scratch_dir)
with utils.MountFileSystem(devices[0], mount_point, self._fs_type):
logging.info('Copying contents')
#feoff: temporary disable selinux to ensure rsync works fine, see https://github.com/GoogleCloudPlatform/compute-image-packages/issues/132
selinux_state = self._setSELinux("0")
self._CopySourceFiles(mount_point)
self._CopyPlatformSpecialFiles(mount_point)
self._ProcessOverwriteList(mount_point)
self._CleanupNetwork(mount_point)
self._UpdateFstab(mount_point, uuid)
# feoff: set dhcp to eth0
self._SetDhcp(mount_point)
# feoff: grub
# should move add_grub to the parm
add_grub = True
if add_grub:
from gcimagebundlelib import grub
grub.InstallGrub(mount_point, devices[0])
self._setSELinux(selinux_state)
tar_entries = []
manifest_file_path = os.path.join(self._scratch_dir, 'manifest.json')
manifest_created = self._manifest.CreateIfNeeded(manifest_file_path)
if manifest_created:
tar_entries.append(manifest_file_path)
tar_entries.append(disk_file_path)
#TODO(feoff): make it parametrizable - if to start tar or not
# logging.info('Creating tar.gz archive')
utils.TarAndGzipFile(tar_entries,
self._output_tarfile)
# Removed the deletion of file
for tar_entry in tar_entries:
if not self._output_tarfile == tar_entry:
os.remove(tar_entry)
# TODO(user): It would be better to compute tar.gz file hash during
# archiving.
h = hashlib.sha1()
with open(self._output_tarfile, 'rb') as tar_file:
for chunk in iter(lambda: tar_file.read(8192), ''):
h.update(chunk)
return (self._fs_size, h.hexdigest())
def Bundleup(self):
"""Creates a raw disk copy of OS image and bundles it into gzipped tar.
Returns:
A size of a generated raw disk and the SHA1 digest of the the tar archive.
Raises:
RawDiskError: If number of partitions in a created image doesn't match
expected count.
"""
# Create sparse file with specified size
disk_file_path = os.path.join(self._scratch_dir, 'disk.raw')
with open(disk_file_path, 'wb') as _:
pass
self._excludes.append(exclude_spec.ExcludeSpec(disk_file_path))
logging.info('Initializing disk file')
partition_start = None
uuid = None
if self._disk:
# If a disk device has been provided then preserve whatever is there on
# the disk before the first partition in case there is an MBR present.
partition_start, uuid = self._InitializeDiskFileFromDevice(
disk_file_path)
else:
# User didn't specify a disk device. Initialize a device with a simple
# partition table.
self._ResizeFile(disk_file_path, self._fs_size)
# User didn't specify a disk to copy. Create a new partition table
utils.MakePartitionTable(disk_file_path)
# Pass 1MB as start to avoid 'Warning: The resulting partition is not
# properly aligned for best performance.' from parted.
partition_start = 1024 * 1024
# Create a new partition starting at partition_start of size
# self._fs_size - partition_start
utils.MakePartition(disk_file_path, 'primary', 'ext2', partition_start,
self._fs_size - partition_start)
with utils.LoadDiskImage(disk_file_path) as devices:
# For now we only support disks with a single partition.
if len(devices) != 1:
raise RawDiskError(devices)
# List contents of /dev/mapper to help with debugging. Contents will
# be listed in debug log only
utils.RunCommand(['ls', '/dev/mapper'])
logging.info('Making filesystem')
uuid = utils.MakeFileSystem(devices[0], self._fs_type, uuid)
with utils.LoadDiskImage(disk_file_path) as devices:
if uuid is None:
raise Exception('Could not get uuid from MakeFileSystem')
mount_point = tempfile.mkdtemp(dir=self._scratch_dir)
with utils.MountFileSystem(devices[0], mount_point):
logging.info('Copying contents')
self._CopySourceFiles(mount_point)
self._CopyPlatformSpecialFiles(mount_point)
self._ProcessOverwriteList(mount_point)
self._CleanupNetwork(mount_point)
self._UpdateFstab(mount_point, uuid)
tar_entries = []
manifest_file_path = os.path.join(self._scratch_dir, 'manifest.json')
manifest_created = self._manifest.CreateIfNeeded(manifest_file_path)
if manifest_created:
tar_entries.append(manifest_file_path)
tar_entries.append(disk_file_path)
logging.info('Creating tar.gz archive')
utils.TarAndGzipFile(tar_entries, self._output_tarfile)
for tar_entry in tar_entries:
os.remove(tar_entry)
# TODO(user): It would be better to compute tar.gz file hash during
# archiving.
h = hashlib.sha1()
with open(self._output_tarfile, 'rb') as tar_file:
for chunk in iter(lambda: tar_file.read(8192), ''):
h.update(chunk)
return (self._fs_size, h.hexdigest())
def RunCommandUnderTest(): non_existent_path = '/' + uuid.uuid4().hex utils.RunCommand(['mkfs', '-t', 'ext4', non_existent_path])
def testRunCommand(self): """Run a simple command and verify it works.""" utils.RunCommand(['ls', '/'])
