def setUp(self):
super(FsRawDiskTest, self).setUp()
self._bundle = block_disk.FsRawDisk(10 * 1024 * 1024)
self._tar_path = self.tmp_path + '/image.tar.gz'
self._bundle.SetTarfile(self._tar_path)
self._bundle.AppendExcludes([exclude_spec.ExcludeSpec(self._tar_path)])
self._bundle.SetKey('key')
def testRawDiskIgnoresExcludes(self):
"""Tests if the raw disk ignores specified excludes files."""
self._bundle.AddSource(self.tmp_path)
self._bundle.AppendExcludes(
[exclude_spec.ExcludeSpec(self.tmp_path + '/dir1')])
self._bundle.Verify()
(_, digest) = self._bundle.Bundleup()
if not digest:
self.fail('raw disk failed')
self._VerifyTarHas(self._tar_path, ['disk.raw'])
self._VerifyImageHas(self._tar_path, [
'lost+found', 'test1', 'test2', 'dir2/', '/dir2/dir1', '/dir2/sl2',
'/dir2/hl1'
])
def testRawDiskHonorsRecursiveOff(self):
"""Tests if raw disk handles recursive off."""
self._bundle.AppendExcludes([exclude_spec.ExcludeSpec(self._tar_path)])
self._bundle.AddSource(self.tmp_path + '/dir1',
arcname='dir1',
recursive=False)
self._bundle.AddSource(self.tmp_path + '/dir2', arcname='dir2')
self._bundle.Verify()
(_, digest) = self._bundle.Bundleup()
if not digest:
self.fail('raw disk failed')
self._VerifyTarHas(self._tar_path, ['disk.raw'])
self._VerifyImageHas(self._tar_path, [
'lost+found', 'dir1/', 'dir2/', '/dir2/dir1', '/dir2/sl2',
'/dir2/hl1'
])
def testRawDiskExcludePreservesSubdirs(self):
"""Tests if excludes preserves subdirs underneath if asked."""
self._bundle.AddSource(self.tmp_path)
self._bundle.AppendExcludes([
exclude_spec.ExcludeSpec(self.tmp_path + '/dir1',
preserve_dir=True,
preserve_subdir=True)
])
self._bundle.Verify()
(_, digest) = self._bundle.Bundleup()
if not digest:
self.fail('raw disk failed')
self._VerifyTarHas(self._tar_path, ['disk.raw'])
self._VerifyImageHas(self._tar_path, [
'lost+found', 'test1', 'test2', 'dir1/', '/dir1/dir11', 'dir2/',
'/dir2/dir1', '/dir2/sl2', '/dir2/hl1'
])
def Bundleup(self):
"""Creates a raw disk copy of OS image and bundles it into gzipped tar.
Returns:
A size of a generated raw disk and the SHA1 digest of the the tar archive.
Raises:
RawDiskError: If number of partitions in a created image doesn't match
expected count.
"""
self._Verify()
# Create sparse file with specified size
file_path = os.path.join(self._scratch_dir, 'disk.raw')
self._excludes.append(exclude_spec.ExcludeSpec(file_path))
with open(file_path, 'wb') as disk_file:
disk_file.truncate(self._fs_size)
utils.MakePartitionTable(file_path)
# Pass 1MB as start to avoid 'Warning: The resulting partition is not
# properly aligned for best performance.' from parted.
utils.MakePartition(file_path, 'primary', 'ext2', 1024 * 1024,
self._fs_size)
with utils.LoadDiskImage(file_path) as devices:
# For now we only support disks with a single partition.
if len(devices) != 1:
raise RawDiskError(devices)
uuid = utils.MakeFileSystem(devices[0], 'ext4')
if uuid is None:
raise Exception('Could not get uuid from makefilesystem')
mount_point = tempfile.mkdtemp(dir=self._scratch_dir)
with utils.MountFileSystem(devices[0], mount_point):
self._CopySourceFiles(mount_point)
self._CopyPlatformSpecialFiles(mount_point)
self._ProcessOverwriteList(mount_point)
self._CleanupNetwork(mount_point)
self._UpdateFstab(mount_point, uuid)
utils.TarAndGzipFile(file_path, self._output_tarfile)
os.remove(file_path)
# TODO(user): It would be better to compute tar.gz file hash during
# archiving.
h = hashlib.sha1()
with open(self._output_tarfile, 'rb') as tar_file:
for chunk in iter(lambda: tar_file.read(8192), ''):
h.update(chunk)
return (self._fs_size, h.hexdigest())
def testRawDiskUsesModifiedFiles(self):
"""Tests if the raw disk uses modified files."""
self._bundle.AddSource(self.tmp_path)
self._bundle.AppendExcludes(
[exclude_spec.ExcludeSpec(self.tmp_path + '/dir1')])
self._bundle.SetPlatform(
image_bundle_test_base.MockPlatform(self.tmp_root))
self._bundle.Verify()
(_, digest) = self._bundle.Bundleup()
if not digest:
self.fail('raw disk failed')
self._VerifyTarHas(self._tar_path, ['disk.raw'])
self._VerifyImageHas(self._tar_path, [
'lost+found', 'test1', 'test2', 'dir2/', '/dir2/dir1', '/dir2/sl2',
'/dir2/hl1'
])
self._VerifyFileInRawDiskEndsWith(self._tar_path, 'test1',
'something extra.')
class LinuxPlatform(os_platform.Platform):
"""Base class for all Linux flavors."""
EXCLUDE_LIST = [
exclude_spec.ExcludeSpec('/tmp', preserve_dir=True),
exclude_spec.ExcludeSpec('/var/log',
preserve_dir=True,
preserve_subdir=True),
exclude_spec.ExcludeSpec('/etc/ssh/.host_key_regenerated'),
exclude_spec.ExcludeSpec('/var/run', preserve_dir=True),
exclude_spec.ExcludeSpec('/var/lib/google/per-instance',
preserve_dir=True)
]
def __init__(self):
"""Populate the uname -a information."""
super(LinuxPlatform, self).__init__()
(self.name, self.hostname, self.release, self.version,
self.architecture, self.processor) = platform.uname()
(self.distribution, self.distribution_version,
self.distribution_codename) = platform.dist()
def GetPlatformDetails(self):
return ' '.join([
self.name, self.hostname, self.release, self.version,
self.architecture, self.processor, self.distribution,
self.distribution_version, self.distribution_codename
])
def GetName(self):
return self.GetOs()
def GetProcessor(self):
return platform.processor()
def GetArchitecture(self):
if self.architecture:
return self.architecture
return ''
def GetOs(self):
if self.distribution:
if self.distribution_codename:
return '%s (%s)' % (self.distribution,
self.distribution_codename)
else:
return self.distribution
if self.name:
return self.name
return 'Linux'
def IsLinux(self):
return True
# Linux specific methods
def GetKernelVersion(self):
return self.release
# distribution specific methods
# if platforms module does not do a good job override these.
def GetDistribution(self):
return self.distribution
def GetDistributionCodeName(self):
return self.distribution_codename
def GetDistributionVersion(self):
return self.distribution_version
def GetPlatformSpecialFiles(self, tmpdir='/tmp'):
"""Creates any platform specific special files."""
retval = []
console_dev = os.makedev(5, 1)
os.mknod(tmpdir + 'console',
stat.S_IFCHR | stat.S_IRUSR | stat.S_IWUSR, console_dev)
retval.append((tmpdir + 'console', 'dev/console'))
null_dev = os.makedev(1, 3)
os.mknod(
tmpdir + 'null', stat.S_IFCHR | stat.S_IRUSR | stat.S_IWUSR
| stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH,
null_dev)
retval.append((tmpdir + 'null', 'dev/null'))
tty_dev = os.makedev(5, 0)
os.mknod(
tmpdir + 'tty', stat.S_IFCHR | stat.S_IRUSR | stat.S_IWUSR
| stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH,
tty_dev)
retval.append((tmpdir + 'tty', 'dev/tty'))
zero_dev = os.makedev(1, 5)
os.mknod(
tmpdir + 'zero', stat.S_IFCHR | stat.S_IRUSR | stat.S_IWUSR
| stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH,
zero_dev)
retval.append((tmpdir + 'zero', 'dev/zero'))
return retval
def Overwrite(self, filename, arcname, tmpdir='/tmp'):
"""Overwrites specified file if needed for the Linux platform."""
pass
def main():
#EnsureSuperUser()
parser = SetupArgsParser()
(options, _) = parser.parse_args()
VerifyArgs(parser, options)
if options.display_version:
PrintVersionInfo()
return 0
scratch_dir = tempfile.mkdtemp(dir=options.output_directory)
SetupLogging(options, scratch_dir)
try:
guest_platform = platform_factory.PlatformFactory(
options.root_directory).GetPlatform()
except platform_factory.UnknownPlatformException:
print 'Could not determine host platform try -s option.'
return -1
temp_file_name = tempfile.mktemp(dir=scratch_dir, suffix='.tar.gz')
bundle = block_disk.RootFsRaw(options.fs_size)
bundle.SetTarfile(temp_file_name)
bundle.AddSource(options.root_directory)
bundle.SetKey(options.key)
bundle.SetScratchDirectory(scratch_dir)
# Merge platform specific exclude list, mounts points
# and user specified excludes
excludes = guest_platform.GetExcludeList()
if options.excludes:
excludes.extend(
[exclude_spec.ExcludeSpec(x) for x in options.excludes.split(',')])
logging.info('exclude list: %s', ' '.join([x.GetSpec() for x in excludes]))
bundle.AppendExcludes(excludes)
if not options.include_mounts:
mount_points = utils.GetMounts(options.root_directory)
logging.info('ignoring mounts %s', ' '.join(mount_points))
bundle.AppendExcludes([
exclude_spec.ExcludeSpec(x, preserve_dir=True)
for x in utils.GetMounts(options.root_directory)
])
bundle.SetPlatform(guest_platform)
# Verify that bundle attributes are correct and create tar bundle.
bundle.Verify()
(fs_size, digest) = bundle.Bundleup()
if not digest:
logging.critical('Could not get digest for the bundle.'
' The bundle may not be created correctly')
return -1
if fs_size > options.fs_size:
logging.critical('Size of tar %d exceeds the file system size %d.',
fs_size, options.fs_size)
return -1
if options.output_file_name:
output_file = os.path.join(options.output_directory,
options.output_file_name)
else:
output_file = os.path.join(options.output_directory,
'%s.image.tar.gz' % digest)
os.rename(temp_file_name, output_file)
if options.bucket:
bucket = options.bucket
if bucket.startswith('gs://'):
output_bucket = '%s/%s' % (bucket, os.path.basename(output_file))
else:
output_bucket = 'gs://%s/%s' % (bucket,
os.path.basename(output_file))
# TODO: Consider using boto library directly.
cmd = ['gsutil', 'cp', output_file, output_bucket]
retcode = subprocess.call(cmd)
if retcode != 0:
logging.critical(
'Failed to copy image to bucket. '
'gsutil returned %d. To retry, run the command: %s', retcode,
' '.join(cmd))
return -1
logging.info('Uploaded image to %s', output_bucket)
# If we've uploaded, then we can remove the local file.
os.remove(output_file)
if options.cleanup:
shutil.rmtree(scratch_dir)