def _TestDiskBundleHelper(self, partition_start, partition_end, fs_uuid): disk_path = self._SetupMbrDisk(partition_start, partition_end, fs_uuid) with utils.LoadDiskImage(disk_path) as devices: # Get the path to do the disk. # devices will have something which is like /dev/mapper/loop0p1 # We need to get loop0 out of it. disk_loop_back_path = '/dev/' + devices[0].split('/')[3][:-2] # Create a symlinks to the disk and loopback paths # This is required because of the code where we assume first # partition is device path appended by 1. Will remove it once we # update that part of the code. symlink_disk = os.path.join(self.tmp_root, 'disk') symlink_partition = self.tmp_root + '/disk1' utils.RunCommand(['ln', '-s', disk_loop_back_path, symlink_disk]) utils.RunCommand(['ln', '-s', devices[0], symlink_partition]) # Bundle up self._bundle.AddDisk(symlink_disk) self._bundle.AddSource(self.tmp_path) self._bundle.Verify() (_, _) = self._bundle.Bundleup() self._VerifyImageHas(self._tar_path, [ 'lost+found', 'test1', 'test2', 'dir1/', '/dir1/dir11/', '/dir1/sl1', '/dir1/hl2', 'dir2/', '/dir2/dir1', '/dir2/sl2', '/dir2/hl1' ]) self._VerifyNumberOfHardLinksInRawDisk(self._tar_path, 'test1', 2) self._VerifyNumberOfHardLinksInRawDisk(self._tar_path, 'test2', 2) self._VerifyDiskSize(self._tar_path, self._fs_size) self._VerifyNonPartitionContents(self._tar_path, disk_path, partition_start) self._VerifyFilesystemUUID(self._tar_path, fs_uuid)
def testDiskBundle(self): """Tests bundle command when a disk is specified. Creates a 20Gb source disk to start with and verifies that creating a 10MB file off it works. """ # Create a 20GB disk with first partition starting at 1MB self._TestDiskBundleHelper(FsRawDiskTest._MEGABYTE, FsRawDiskTest._GIGABYTE * 20, utils.RunCommand(['uuidgen']).strip())
def testDiskBundlePartitionAt2MB(self): """Tests bundle command when a disk is specified. Creates the first partition at 2MB and verifies all data prior to that is copied. """ # Create a 20GB disk with first partition starting at 2MB self._TestDiskBundleHelper(FsRawDiskTest._MEGABYTE * 2, FsRawDiskTest._GIGABYTE * 20, utils.RunCommand(['uuidgen']).strip())
def main(): parser = SetupArgsParser() (options, _) = parser.parse_args() if options.display_version: PrintVersionInfo() return 0 EnsureSuperUser() VerifyArgs(parser, options) scratch_dir = tempfile.mkdtemp(dir=options.output_directory) SetupLogging(options, scratch_dir) logging.warn( '============================================================\n' 'Warning: gcimagebundle is deprecated. See\n' 'https://cloud.google.com/compute/docs/creating-custom-image' '#export_an_image_to_google_cloud_storage\n' 'for updated instructions.\n' '============================================================') try: guest_platform = platform_factory.PlatformFactory( options.root_directory).GetPlatform() except platform_factory.UnknownPlatformException: logging.critical( 'Platform is not supported.' ' Platform rules can be added to platform_factory.py.') return -1 temp_file_name = tempfile.mktemp(dir=scratch_dir, suffix='.tar.gz') file_system = GetTargetFilesystem(options, guest_platform) logging.info('File System: %s', file_system) logging.info('Disk Size: %s bytes', options.fs_size) bundle = block_disk.RootFsRaw(options.fs_size, file_system, options.skip_disk_space_check) bundle.SetTarfile(temp_file_name) if options.disk: readlink_command = ['readlink', '-f', options.disk] final_path = utils.RunCommand(readlink_command).strip() logging.info('Resolved %s to %s', options.disk, final_path) bundle.AddDisk(final_path) # TODO(user): Find the location where the first partition of the disk # is mounted and add it as the source instead of relying on the source # param flag bundle.AddSource(options.root_directory) bundle.SetKey(options.key) bundle.SetScratchDirectory(scratch_dir) # Merge platform specific exclude list, mounts points # and user specified excludes excludes = guest_platform.GetExcludeList() if options.excludes: excludes.extend( [exclude_spec.ExcludeSpec(x) for x in options.excludes.split(',')]) logging.info('exclude list: %s', ' '.join([x.GetSpec() for x in excludes])) bundle.AppendExcludes(excludes) if not options.include_mounts: mount_points = utils.GetMounts(options.root_directory) logging.info('ignoring mounts %s', ' '.join(mount_points)) bundle.AppendExcludes([ exclude_spec.ExcludeSpec(x, preserve_dir=True) for x in utils.GetMounts(options.root_directory) ]) bundle.SetPlatform(guest_platform) # Verify that bundle attributes are correct and create tar bundle. bundle.Verify() (fs_size, digest) = bundle.Bundleup() if not digest: logging.critical('Could not get digest for the bundle.' ' The bundle may not be created correctly') return -1 if fs_size > options.fs_size: logging.critical('Size of tar %d exceeds the file system size %d.', fs_size, options.fs_size) return -1 if options.output_file_name: output_file = os.path.join(options.output_directory, options.output_file_name) else: output_file = os.path.join(options.output_directory, '%s.image.tar.gz' % digest) os.rename(temp_file_name, output_file) logging.info('Created tar.gz file at %s' % output_file) if options.bucket: bucket = options.bucket if bucket.startswith('gs://'): output_bucket = '%s/%s' % (bucket, os.path.basename(output_file)) else: output_bucket = 'gs://%s/%s' % (bucket, os.path.basename(output_file)) # /usr/local/bin not in redhat root PATH by default if '/usr/local/bin' not in os.environ['PATH']: os.environ['PATH'] += ':/usr/local/bin' # TODO: Consider using boto library directly. cmd = ['gsutil', 'cp', output_file, output_bucket] retcode = subprocess.call(cmd) if retcode != 0: logging.critical( 'Failed to copy image to bucket. ' 'gsutil returned %d. To retry, run the command: %s', retcode, ' '.join(cmd)) return -1 logging.info('Uploaded image to %s', output_bucket) # If we've uploaded, then we can remove the local file. os.remove(output_file) if options.cleanup: shutil.rmtree(scratch_dir)
def Bundleup(self): """Creates a raw disk copy of OS image and bundles it into gzipped tar. Returns: A size of a generated raw disk and the SHA1 digest of the the tar archive. Raises: RawDiskError: If number of partitions in a created image doesn't match expected count. """ # Create sparse file with specified size disk_file_path = os.path.join(self._scratch_dir, self._disk_file_name) # with open(disk_file_path, 'wb') as _: # pass self._excludes.append(exclude_spec.ExcludeSpec(disk_file_path)) logging.info('Initializing disk file') partition_start = None uuid = None if self._disk: # If a disk device has been provided then preserve whatever is there on # the disk before the first partition in case there is an MBR present. partition_start, uuid = self._InitializeDiskFileFromDevice(disk_file_path) else: #feoff: our code goes this way # User didn't specify a disk device. Initialize a device with a simple # partition table. logging.info("Setting partiton size: " + str(self._fs_size) ) self._ResizeFile(disk_file_path, self._fs_size) # User didn't specify a disk to copy. Create a new partition table utils.MakePartitionTable(disk_file_path) # Pass 1MB as start to avoid 'Warning: The resulting partition is not # properly aligned for best performance.' from parted. partition_start = 1024 * 1024 # Create a new partition starting at partition_start of size # self._fs_size - partition_start. #feoff: it's not just a log ensure the size changed. we ensure the size is changed and OS updated info on it. needed in FUSE scenario utils.MakePartition(disk_file_path, 'primary', 'ext2', partition_start, self._fs_size-512) # feoff: THE LAST PARM IS THE LAST SECTOR! , not size! logging.info("Preparing file disk size " + str(os.path.getsize(disk_file_path)) + " bytes") with utils.LoadDiskImage(disk_file_path) as devices: # For now we only support disks with a single partition. if len(devices) != 1: raise RawDiskError(devices) # List contents of /dev/mapper to help with debugging. Contents will # be listed in debug log only utils.RunCommand(['ls', '/dev/mapper']) logging.info('Making filesystem') uuid = utils.MakeFileSystem(devices[0], self._fs_type, uuid) with utils.LoadDiskImage(disk_file_path) as devices: if uuid is None: raise Exception('Could not get uuid from MakeFileSystem') mount_point = tempfile.mkdtemp(dir=self._scratch_dir) with utils.MountFileSystem(devices[0], mount_point, self._fs_type): logging.info('Copying contents') #feoff: temporary disable selinux to ensure rsync works fine, see https://github.com/GoogleCloudPlatform/compute-image-packages/issues/132 selinux_state = self._setSELinux("0") self._CopySourceFiles(mount_point) self._CopyPlatformSpecialFiles(mount_point) self._ProcessOverwriteList(mount_point) self._CleanupNetwork(mount_point) self._UpdateFstab(mount_point, uuid) # feoff: set dhcp to eth0 self._SetDhcp(mount_point) # feoff: grub # should move add_grub to the parm add_grub = True if add_grub: from gcimagebundlelib import grub grub.InstallGrub(mount_point, devices[0]) self._setSELinux(selinux_state) tar_entries = [] manifest_file_path = os.path.join(self._scratch_dir, 'manifest.json') manifest_created = self._manifest.CreateIfNeeded(manifest_file_path) if manifest_created: tar_entries.append(manifest_file_path) tar_entries.append(disk_file_path) #TODO(feoff): make it parametrizable - if to start tar or not # logging.info('Creating tar.gz archive') utils.TarAndGzipFile(tar_entries, self._output_tarfile) # Removed the deletion of file for tar_entry in tar_entries: if not self._output_tarfile == tar_entry: os.remove(tar_entry) # TODO(user): It would be better to compute tar.gz file hash during # archiving. h = hashlib.sha1() with open(self._output_tarfile, 'rb') as tar_file: for chunk in iter(lambda: tar_file.read(8192), ''): h.update(chunk) return (self._fs_size, h.hexdigest())
def Bundleup(self): """Creates a raw disk copy of OS image and bundles it into gzipped tar. Returns: A size of a generated raw disk and the SHA1 digest of the the tar archive. Raises: RawDiskError: If number of partitions in a created image doesn't match expected count. """ # Create sparse file with specified size disk_file_path = os.path.join(self._scratch_dir, 'disk.raw') with open(disk_file_path, 'wb') as _: pass self._excludes.append(exclude_spec.ExcludeSpec(disk_file_path)) logging.info('Initializing disk file') partition_start = None uuid = None if self._disk: # If a disk device has been provided then preserve whatever is there on # the disk before the first partition in case there is an MBR present. partition_start, uuid = self._InitializeDiskFileFromDevice( disk_file_path) else: # User didn't specify a disk device. Initialize a device with a simple # partition table. self._ResizeFile(disk_file_path, self._fs_size) # User didn't specify a disk to copy. Create a new partition table utils.MakePartitionTable(disk_file_path) # Pass 1MB as start to avoid 'Warning: The resulting partition is not # properly aligned for best performance.' from parted. partition_start = 1024 * 1024 # Create a new partition starting at partition_start of size # self._fs_size - partition_start utils.MakePartition(disk_file_path, 'primary', 'ext2', partition_start, self._fs_size - partition_start) with utils.LoadDiskImage(disk_file_path) as devices: # For now we only support disks with a single partition. if len(devices) != 1: raise RawDiskError(devices) # List contents of /dev/mapper to help with debugging. Contents will # be listed in debug log only utils.RunCommand(['ls', '/dev/mapper']) logging.info('Making filesystem') uuid = utils.MakeFileSystem(devices[0], self._fs_type, uuid) with utils.LoadDiskImage(disk_file_path) as devices: if uuid is None: raise Exception('Could not get uuid from MakeFileSystem') mount_point = tempfile.mkdtemp(dir=self._scratch_dir) with utils.MountFileSystem(devices[0], mount_point): logging.info('Copying contents') self._CopySourceFiles(mount_point) self._CopyPlatformSpecialFiles(mount_point) self._ProcessOverwriteList(mount_point) self._CleanupNetwork(mount_point) self._UpdateFstab(mount_point, uuid) tar_entries = [] manifest_file_path = os.path.join(self._scratch_dir, 'manifest.json') manifest_created = self._manifest.CreateIfNeeded(manifest_file_path) if manifest_created: tar_entries.append(manifest_file_path) tar_entries.append(disk_file_path) logging.info('Creating tar.gz archive') utils.TarAndGzipFile(tar_entries, self._output_tarfile) for tar_entry in tar_entries: os.remove(tar_entry) # TODO(user): It would be better to compute tar.gz file hash during # archiving. h = hashlib.sha1() with open(self._output_tarfile, 'rb') as tar_file: for chunk in iter(lambda: tar_file.read(8192), ''): h.update(chunk) return (self._fs_size, h.hexdigest())
def RunCommandUnderTest(): non_existent_path = '/' + uuid.uuid4().hex utils.RunCommand(['mkfs', '-t', 'ext4', non_existent_path])
def testRunCommand(self): """Run a simple command and verify it works.""" utils.RunCommand(['ls', '/'])