def setUp(self): super(RootFsRawTest, self).setUp() self._bundle = block_disk.RootFsRaw(10 * 1024 * 1024, 'ext4', False, self._MockStatvfs) self._tar_path = self.tmp_path + '/image.tar.gz' self._bundle.SetTarfile(self._tar_path) self._bundle.AppendExcludes([exclude_spec.ExcludeSpec(self._tar_path)]) self._bundle._SetManifest(self._manifest)
def setUp(self): super(FsRawDiskTest, self).setUp() self._fs_size = 10 * FsRawDiskTest._MEGABYTE self._bundle = block_disk.FsRawDisk(self._fs_size, 'ext4') self._tar_path = self.tmp_path + '/image.tar.gz' self._bundle.SetTarfile(self._tar_path) self._bundle.AppendExcludes([exclude_spec.ExcludeSpec(self._tar_path)]) self._bundle.SetKey('key') self._bundle._SetManifest(self._manifest)
def testRawDiskIgnoresExcludes(self): """Tests if the raw disk ignores specified excludes files.""" self._bundle.AddSource(self.tmp_path) self._bundle.AppendExcludes( [exclude_spec.ExcludeSpec(self.tmp_path + '/dir1')]) self._bundle.Verify() (_, digest) = self._bundle.Bundleup() if not digest: self.fail('raw disk failed') self._VerifyTarHas(self._tar_path, ['disk.raw']) self._VerifyImageHas(self._tar_path, [ 'lost+found', 'test1', 'test2', 'dir2/', '/dir2/dir1', '/dir2/sl2', '/dir2/hl1' ])
def testRawDiskHonorsRecursiveOff(self): """Tests if raw disk handles recursive off.""" self._bundle.AppendExcludes([exclude_spec.ExcludeSpec(self._tar_path)]) self._bundle.AddSource(self.tmp_path + '/dir1', arcname='dir1', recursive=False) self._bundle.AddSource(self.tmp_path + '/dir2', arcname='dir2') self._bundle.Verify() (_, digest) = self._bundle.Bundleup() if not digest: self.fail('raw disk failed') self._VerifyTarHas(self._tar_path, ['disk.raw']) self._VerifyImageHas(self._tar_path, [ 'lost+found', 'dir1/', 'dir2/', '/dir2/dir1', '/dir2/sl2', '/dir2/hl1' ])
def testRawDiskExcludePreservesFiles(self): """Tests if excludes preserves the files underneath if asked.""" self._bundle.AddSource(self.tmp_path) self._bundle.AppendExcludes([ exclude_spec.ExcludeSpec(self.tmp_path + '/dir1', preserve_dir=True, preserve_file=True) ]) self._bundle.Verify() (_, digest) = self._bundle.Bundleup() if not digest: self.fail('raw disk failed') self._VerifyTarHas(self._tar_path, ['disk.raw']) self._VerifyImageHas(self._tar_path, [ 'lost+found', 'test1', 'test2', 'dir1/', '/dir1/hl2', '/dir1/sl1', 'dir2/', '/dir2/dir1', '/dir2/sl2', '/dir2/hl1' ])
def testRawDiskUsesModifiedFiles(self): """Tests if the raw disk uses modified files.""" self._bundle.AddSource(self.tmp_path) self._bundle.AppendExcludes( [exclude_spec.ExcludeSpec(self.tmp_path + '/dir1')]) self._bundle.SetPlatform( image_bundle_test_base.MockPlatform(self.tmp_root)) self._bundle.Verify() (_, digest) = self._bundle.Bundleup() if not digest: self.fail('raw disk failed') self._VerifyTarHas(self._tar_path, ['disk.raw']) self._VerifyImageHas(self._tar_path, [ 'lost+found', 'test1', 'test2', 'dir2/', '/dir2/dir1', '/dir2/sl2', '/dir2/hl1' ]) self._VerifyFileInRawDiskEndsWith(self._tar_path, 'test1', 'something extra.')
def main(): parser = SetupArgsParser() (options, _) = parser.parse_args() if options.display_version: PrintVersionInfo() return 0 EnsureSuperUser() VerifyArgs(parser, options) scratch_dir = tempfile.mkdtemp(dir=options.output_directory) SetupLogging(options, scratch_dir) logging.warn( '============================================================\n' 'Warning: gcimagebundle is deprecated. See\n' 'https://cloud.google.com/compute/docs/creating-custom-image' '#export_an_image_to_google_cloud_storage\n' 'for updated instructions.\n' '============================================================') try: guest_platform = platform_factory.PlatformFactory( options.root_directory).GetPlatform() except platform_factory.UnknownPlatformException: logging.critical( 'Platform is not supported.' ' Platform rules can be added to platform_factory.py.') return -1 temp_file_name = tempfile.mktemp(dir=scratch_dir, suffix='.tar.gz') file_system = GetTargetFilesystem(options, guest_platform) logging.info('File System: %s', file_system) logging.info('Disk Size: %s bytes', options.fs_size) bundle = block_disk.RootFsRaw(options.fs_size, file_system, options.skip_disk_space_check) bundle.SetTarfile(temp_file_name) if options.disk: readlink_command = ['readlink', '-f', options.disk] final_path = utils.RunCommand(readlink_command).strip() logging.info('Resolved %s to %s', options.disk, final_path) bundle.AddDisk(final_path) # TODO(user): Find the location where the first partition of the disk # is mounted and add it as the source instead of relying on the source # param flag bundle.AddSource(options.root_directory) bundle.SetKey(options.key) bundle.SetScratchDirectory(scratch_dir) # Merge platform specific exclude list, mounts points # and user specified excludes excludes = guest_platform.GetExcludeList() if options.excludes: excludes.extend( [exclude_spec.ExcludeSpec(x) for x in options.excludes.split(',')]) logging.info('exclude list: %s', ' '.join([x.GetSpec() for x in excludes])) bundle.AppendExcludes(excludes) if not options.include_mounts: mount_points = utils.GetMounts(options.root_directory) logging.info('ignoring mounts %s', ' '.join(mount_points)) bundle.AppendExcludes([ exclude_spec.ExcludeSpec(x, preserve_dir=True) for x in utils.GetMounts(options.root_directory) ]) bundle.SetPlatform(guest_platform) # Verify that bundle attributes are correct and create tar bundle. bundle.Verify() (fs_size, digest) = bundle.Bundleup() if not digest: logging.critical('Could not get digest for the bundle.' ' The bundle may not be created correctly') return -1 if fs_size > options.fs_size: logging.critical('Size of tar %d exceeds the file system size %d.', fs_size, options.fs_size) return -1 if options.output_file_name: output_file = os.path.join(options.output_directory, options.output_file_name) else: output_file = os.path.join(options.output_directory, '%s.image.tar.gz' % digest) os.rename(temp_file_name, output_file) logging.info('Created tar.gz file at %s' % output_file) if options.bucket: bucket = options.bucket if bucket.startswith('gs://'): output_bucket = '%s/%s' % (bucket, os.path.basename(output_file)) else: output_bucket = 'gs://%s/%s' % (bucket, os.path.basename(output_file)) # /usr/local/bin not in redhat root PATH by default if '/usr/local/bin' not in os.environ['PATH']: os.environ['PATH'] += ':/usr/local/bin' # TODO: Consider using boto library directly. cmd = ['gsutil', 'cp', output_file, output_bucket] retcode = subprocess.call(cmd) if retcode != 0: logging.critical( 'Failed to copy image to bucket. ' 'gsutil returned %d. To retry, run the command: %s', retcode, ' '.join(cmd)) return -1 logging.info('Uploaded image to %s', output_bucket) # If we've uploaded, then we can remove the local file. os.remove(output_file) if options.cleanup: shutil.rmtree(scratch_dir)
def Bundleup(self): """Creates a raw disk copy of OS image and bundles it into gzipped tar. Returns: A size of a generated raw disk and the SHA1 digest of the the tar archive. Raises: RawDiskError: If number of partitions in a created image doesn't match expected count. """ # Create sparse file with specified size disk_file_path = os.path.join(self._scratch_dir, self._disk_file_name) # with open(disk_file_path, 'wb') as _: # pass self._excludes.append(exclude_spec.ExcludeSpec(disk_file_path)) logging.info('Initializing disk file') partition_start = None uuid = None if self._disk: # If a disk device has been provided then preserve whatever is there on # the disk before the first partition in case there is an MBR present. partition_start, uuid = self._InitializeDiskFileFromDevice(disk_file_path) else: #feoff: our code goes this way # User didn't specify a disk device. Initialize a device with a simple # partition table. logging.info("Setting partiton size: " + str(self._fs_size) ) self._ResizeFile(disk_file_path, self._fs_size) # User didn't specify a disk to copy. Create a new partition table utils.MakePartitionTable(disk_file_path) # Pass 1MB as start to avoid 'Warning: The resulting partition is not # properly aligned for best performance.' from parted. partition_start = 1024 * 1024 # Create a new partition starting at partition_start of size # self._fs_size - partition_start. #feoff: it's not just a log ensure the size changed. we ensure the size is changed and OS updated info on it. needed in FUSE scenario utils.MakePartition(disk_file_path, 'primary', 'ext2', partition_start, self._fs_size-512) # feoff: THE LAST PARM IS THE LAST SECTOR! , not size! logging.info("Preparing file disk size " + str(os.path.getsize(disk_file_path)) + " bytes") with utils.LoadDiskImage(disk_file_path) as devices: # For now we only support disks with a single partition. if len(devices) != 1: raise RawDiskError(devices) # List contents of /dev/mapper to help with debugging. Contents will # be listed in debug log only utils.RunCommand(['ls', '/dev/mapper']) logging.info('Making filesystem') uuid = utils.MakeFileSystem(devices[0], self._fs_type, uuid) with utils.LoadDiskImage(disk_file_path) as devices: if uuid is None: raise Exception('Could not get uuid from MakeFileSystem') mount_point = tempfile.mkdtemp(dir=self._scratch_dir) with utils.MountFileSystem(devices[0], mount_point, self._fs_type): logging.info('Copying contents') #feoff: temporary disable selinux to ensure rsync works fine, see https://github.com/GoogleCloudPlatform/compute-image-packages/issues/132 selinux_state = self._setSELinux("0") self._CopySourceFiles(mount_point) self._CopyPlatformSpecialFiles(mount_point) self._ProcessOverwriteList(mount_point) self._CleanupNetwork(mount_point) self._UpdateFstab(mount_point, uuid) # feoff: set dhcp to eth0 self._SetDhcp(mount_point) # feoff: grub # should move add_grub to the parm add_grub = True if add_grub: from gcimagebundlelib import grub grub.InstallGrub(mount_point, devices[0]) self._setSELinux(selinux_state) tar_entries = [] manifest_file_path = os.path.join(self._scratch_dir, 'manifest.json') manifest_created = self._manifest.CreateIfNeeded(manifest_file_path) if manifest_created: tar_entries.append(manifest_file_path) tar_entries.append(disk_file_path) #TODO(feoff): make it parametrizable - if to start tar or not # logging.info('Creating tar.gz archive') utils.TarAndGzipFile(tar_entries, self._output_tarfile) # Removed the deletion of file for tar_entry in tar_entries: if not self._output_tarfile == tar_entry: os.remove(tar_entry) # TODO(user): It would be better to compute tar.gz file hash during # archiving. h = hashlib.sha1() with open(self._output_tarfile, 'rb') as tar_file: for chunk in iter(lambda: tar_file.read(8192), ''): h.update(chunk) return (self._fs_size, h.hexdigest())
class LinuxPlatform(os_platform.Platform): """Base class for all Linux flavors.""" EXCLUDE_LIST = [ exclude_spec.ExcludeSpec('/etc/ssh/.host_key_regenerated'), exclude_spec.ExcludeSpec('/dev', preserve_dir=True), exclude_spec.ExcludeSpec('/proc', preserve_dir=True), exclude_spec.ExcludeSpec('/run', preserve_dir=True, preserve_subdir=True), exclude_spec.ExcludeSpec('/selinux'), exclude_spec.ExcludeSpec('/tmp', preserve_dir=True), exclude_spec.ExcludeSpec('/sys', preserve_dir=True), exclude_spec.ExcludeSpec('/var/lib/google/per-instance', preserve_dir=True), exclude_spec.ExcludeSpec('/var/lock', preserve_dir=True, preserve_subdir=True), exclude_spec.ExcludeSpec('/var/log', preserve_dir=True, preserve_subdir=True), exclude_spec.ExcludeSpec('/var/run', preserve_dir=True, preserve_subdir=True) ] def __init__(self): """Populate the uname -a information.""" super(LinuxPlatform, self).__init__() (self.name, self.hostname, self.release, self.version, self.architecture, self.processor) = platform.uname() (self.distribution, self.distribution_version, self.distribution_codename) = platform.dist() def GetPlatformDetails(self): return ' '.join([ self.name, self.hostname, self.release, self.version, self.architecture, self.processor, self.distribution, self.distribution_version, self.distribution_codename ]) def GetName(self): return self.GetOs() def GetProcessor(self): return platform.processor() def GetArchitecture(self): if self.architecture: return self.architecture return '' def GetOs(self): if self.distribution: if self.distribution_codename: return '%s (%s)' % (self.distribution, self.distribution_codename) else: return self.distribution if self.name: return self.name return 'Linux' def IsLinux(self): return True # Linux specific methods def GetKernelVersion(self): return self.release # distribution specific methods # if platforms module does not do a good job override these. def GetDistribution(self): return self.distribution def GetDistributionCodeName(self): return self.distribution_codename def GetDistributionVersion(self): return self.distribution_version def GetPlatformSpecialFiles(self, tmpdir='/tmp'): """Creates any platform specific special files.""" retval = [] console_dev = os.makedev(5, 1) os.mknod(tmpdir + 'console', stat.S_IFCHR | stat.S_IRUSR | stat.S_IWUSR, console_dev) retval.append((tmpdir + 'console', 'dev/console')) null_dev = os.makedev(1, 3) os.mknod( tmpdir + 'null', stat.S_IFCHR | stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH, null_dev) retval.append((tmpdir + 'null', 'dev/null')) tty_dev = os.makedev(5, 0) os.mknod( tmpdir + 'tty', stat.S_IFCHR | stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH, tty_dev) retval.append((tmpdir + 'tty', 'dev/tty')) zero_dev = os.makedev(1, 5) os.mknod( tmpdir + 'zero', stat.S_IFCHR | stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH, zero_dev) retval.append((tmpdir + 'zero', 'dev/zero')) # /selinux is deprecated in favor of /sys/fs/selinux, but preserve it on # those OSes where it's present. if os.path.isdir('/selinux'): os.mkdir(tmpdir + 'selinux', 0755) retval.append((tmpdir + 'selinux', 'selinux')) return retval def Overwrite(self, filename, arcname, tmpdir='/tmp'): """Overwrites specified file if needed for the Linux platform.""" pass def GetPreferredFilesystemType(self): """Return the optimal filesystem supported for the platform.""" return 'ext4'
def Bundleup(self): """Creates a raw disk copy of OS image and bundles it into gzipped tar. Returns: A size of a generated raw disk and the SHA1 digest of the the tar archive. Raises: RawDiskError: If number of partitions in a created image doesn't match expected count. """ # Create sparse file with specified size disk_file_path = os.path.join(self._scratch_dir, 'disk.raw') with open(disk_file_path, 'wb') as _: pass self._excludes.append(exclude_spec.ExcludeSpec(disk_file_path)) logging.info('Initializing disk file') partition_start = None uuid = None if self._disk: # If a disk device has been provided then preserve whatever is there on # the disk before the first partition in case there is an MBR present. partition_start, uuid = self._InitializeDiskFileFromDevice( disk_file_path) else: # User didn't specify a disk device. Initialize a device with a simple # partition table. self._ResizeFile(disk_file_path, self._fs_size) # User didn't specify a disk to copy. Create a new partition table utils.MakePartitionTable(disk_file_path) # Pass 1MB as start to avoid 'Warning: The resulting partition is not # properly aligned for best performance.' from parted. partition_start = 1024 * 1024 # Create a new partition starting at partition_start of size # self._fs_size - partition_start utils.MakePartition(disk_file_path, 'primary', 'ext2', partition_start, self._fs_size - partition_start) with utils.LoadDiskImage(disk_file_path) as devices: # For now we only support disks with a single partition. if len(devices) != 1: raise RawDiskError(devices) # List contents of /dev/mapper to help with debugging. Contents will # be listed in debug log only utils.RunCommand(['ls', '/dev/mapper']) logging.info('Making filesystem') uuid = utils.MakeFileSystem(devices[0], self._fs_type, uuid) with utils.LoadDiskImage(disk_file_path) as devices: if uuid is None: raise Exception('Could not get uuid from MakeFileSystem') mount_point = tempfile.mkdtemp(dir=self._scratch_dir) with utils.MountFileSystem(devices[0], mount_point): logging.info('Copying contents') self._CopySourceFiles(mount_point) self._CopyPlatformSpecialFiles(mount_point) self._ProcessOverwriteList(mount_point) self._CleanupNetwork(mount_point) self._UpdateFstab(mount_point, uuid) tar_entries = [] manifest_file_path = os.path.join(self._scratch_dir, 'manifest.json') manifest_created = self._manifest.CreateIfNeeded(manifest_file_path) if manifest_created: tar_entries.append(manifest_file_path) tar_entries.append(disk_file_path) logging.info('Creating tar.gz archive') utils.TarAndGzipFile(tar_entries, self._output_tarfile) for tar_entry in tar_entries: os.remove(tar_entry) # TODO(user): It would be better to compute tar.gz file hash during # archiving. h = hashlib.sha1() with open(self._output_tarfile, 'rb') as tar_file: for chunk in iter(lambda: tar_file.read(8192), ''): h.update(chunk) return (self._fs_size, h.hexdigest())
class LinuxPlatform(os_platform.Platform): """Base class for all Linux flavors.""" EXCLUDE_LIST = [ exclude_spec.ExcludeSpec('/etc/ssh/.host_key_regenerated'), exclude_spec.ExcludeSpec('/dev', preserve_dir=True), exclude_spec.ExcludeSpec('/proc', preserve_dir=True), exclude_spec.ExcludeSpec('/run', preserve_dir=True, preserve_subdir=True), exclude_spec.ExcludeSpec('/selinux'), exclude_spec.ExcludeSpec('/tmp', preserve_dir=True), exclude_spec.ExcludeSpec('/sys', preserve_dir=True), exclude_spec.ExcludeSpec('/var/lib/google/per-instance', preserve_dir=True), exclude_spec.ExcludeSpec('/var/lock', preserve_dir=True, preserve_subdir=True), exclude_spec.ExcludeSpec('/var/log', preserve_dir=True, preserve_subdir=True), exclude_spec.ExcludeSpec('/var/run', preserve_dir=True, preserve_subdir=True) ] UnknownFamily = 0 DebianFamily = 1 RedhatFamily = 2 def __init__(self): """Populate the uname -a information.""" self.linux_family = LinuxPlatform.UnknownFamily super(LinuxPlatform, self).__init__() (self.name, self.hostname, self.release, self.version, self.architecture, self.processor) = platform.uname() (self.distribution, self.distribution_version, self.distribution_codename) = platform.dist() def GetPlatformDetails(self): return ' '.join([ self.name, self.hostname, self.release, self.version, self.architecture, self.processor, self.distribution, self.distribution_version, self.distribution_codename ]) def GetName(self): return self.GetOs() def GetProcessor(self): return platform.processor() def GetArchitecture(self): if self.architecture: return self.architecture return '' def GetOs(self): if self.distribution: if self.distribution_codename: return '%s (%s)' % (self.distribution, self.distribution_codename) else: return self.distribution if self.name: return self.name return 'Linux' def IsLinux(self): return True # Linux specific methods def GetKernelVersion(self): return self.release # distribution specific methods # if platforms module does not do a good job override these. def GetDistribution(self): return self.distribution def GetDistributionCodeName(self): return self.distribution_codename def GetDistributionVersion(self): return self.distribution_version def GetPlatformSpecialFiles(self, tmpdir='/tmp'): """Creates any platform specific special files.""" retval = [] console_dev = os.makedev(5, 1) if (os.path.exists(tmpdir + 'console')): os.remove(tmpdir + 'console') os.mknod(tmpdir + 'console', stat.S_IFCHR | stat.S_IRUSR | stat.S_IWUSR, console_dev) retval.append((tmpdir + 'console', 'dev/console')) null_dev = os.makedev(1, 3) if (os.path.exists(tmpdir + 'null')): os.remove(tmpdir + 'null') os.mknod( tmpdir + 'null', stat.S_IFCHR | stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH, null_dev) retval.append((tmpdir + 'null', 'dev/null')) tty_dev = os.makedev(5, 0) if (os.path.exists(tmpdir + 'tty')): os.remove(tmpdir + 'tty') os.mknod( tmpdir + 'tty', stat.S_IFCHR | stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH, tty_dev) retval.append((tmpdir + 'tty', 'dev/tty')) zero_dev = os.makedev(1, 5) if (os.path.exists(tmpdir + 'zero')): os.remove(tmpdir + 'zero') os.mknod( tmpdir + 'zero', stat.S_IFCHR | stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH, zero_dev) retval.append((tmpdir + 'zero', 'dev/zero')) # /selinux is deprecated in favor of /sys/fs/selinux, but preserve it on # those OSes where it's present. if os.path.isdir('/selinux'): if not (os.path.exists(tmpdir + 'selinux')): os.mkdir(tmpdir + 'selinux', 0755) retval.append((tmpdir + 'selinux', 'selinux')) return retval def Overwrite(self, filename, arcname, tmpdir='/tmp'): """Overwrites specified file if needed for the Linux platform.""" pass def GetPreferredFilesystemType(self): """Return the optimal filesystem supported for the platform.""" return 'ext4' #feoff: added network seetings, they are different for Debian\RHEL distros #TODO: make virtual def GetNetworkSettingsPath(self): """Returns path to netowrk config""" if self.linux_family == LinuxPlatform.RedhatFamily: return "/etc/sysconfig/network-scripts/ifcfg-eth0" if self.linux_family == LinuxPlatform.DebianFamily: return "/etc/network/interfaces" raise NotImplementedError def GetNetworkOptionsString(self, static_ip=None, gateway=None, mask=None): """Returns data to be placed in network config file if static_ip then dhcp enabled config version returned """ if self.linux_family == LinuxPlatform.RedhatFamily: retval = "DEVICE=eth0\n" if static_ip: retval = retval + "BOOTPROTO=static\nDHCPCLASS=\nIPADDR=" + static_ip + "\nNETMASK=" + mask + "\nGATEWAY=" + gateway + "\n" # not sure of gateway else: retval = retval + "BOOTPROTO=dhcp\n" retval = retval + "ONBOOT=yes\n" return retval if self.linux_family == LinuxPlatform.DebianFamily: if static_ip: retval = " #Generated by Cloudscraper\n\nauto lo\n"\ "iface lo inet loopback\n\n" \ "iface eth0 inet static\ \n\taddress" + static_ip + \ "\n\tnetmask" + mask + \ "\n\tgateway" + gateway return retval #"\n\tnetwork" 192.168.1.0 not sure what to add, can be deduced from netmask # broadcast 192.168.1.255 - not sure what to add , can be deduced from netmask else: return "auto eth0\niface eth0 inet dhcp\n" raise NotImplementedError def GetLinuxFamily(self): return self.linux_family