def preflight_check(self):
"""While not all of these are strictly checks, their failure would inevitably
lead to failure, and since we can check them before we start setting up disk
and whatnot, we might as well go ahead an do this now."""
if not self.vm.suite in self.suites:
raise VMBuilderUserError('Invalid suite. Valid suites are: %s' % ' '.join(self.suites))
modname = 'VMBuilder.plugins.ubuntu.%s' % (self.vm.suite, )
mod = __import__(modname, fromlist=[self.vm.suite])
self.suite = getattr(mod, self.vm.suite.capitalize())(self.vm)
if self.vm.arch not in self.valid_archs[self.host_arch] or \
not self.suite.check_arch_validity(self.vm.arch):
raise VMBuilderUserError('%s is not a valid architecture. Valid architectures are: %s' % (self.vm.arch,
' '.join(self.valid_archs[self.host_arch])))
if not self.vm.components:
self.vm.components = ['main', 'restricted', 'universe']
else:
if type(self.vm.components) is str:
self.vm.components = self.vm.components.split(',')
if self.vm.hypervisor.name == 'Xen':
logging.info('Xen kernel default: linux-image-%s %s', self.suite.xen_kernel_flavour, self.xen_kernel_version())
self.vm.virtio_net = self.use_virtio_net()
if self.vm.lang:
try:
run_cmd('locale-gen', '%s' % self.vm.lang)
except VMBuilderException, e:
msg = "locale-gen does not recognize your locale '%s'" % self.vm.lang
raise VMBuilderUserError(msg)
def convert(self, destdir, format):
"""
Convert the disk image
@type destdir: string
@param destdir: Target location of converted disk image
@type format: string
@param format: The target format (as understood by qemu-img or vdi)
@rtype: string
@return: the name of the converted image
"""
if self.preallocated:
# We don't convert preallocated disk images. That would be silly.
return self.filename
filename = os.path.basename(self.filename)
if '.' in filename:
filename = filename[:filename.rindex('.')]
destfile = '%s/%s.%s' % (destdir, filename, format)
logging.info('Converting %s to %s, format %s' %
(self.filename, format, destfile))
if format == 'vdi':
run_cmd(vbox_manager_path(), 'convertfromraw', '-format', 'VDI',
self.filename, destfile)
else:
run_cmd(qemu_img_path(), 'convert', '-O', format, self.filename,
destfile)
os.unlink(self.filename)
self.filename = os.path.abspath(destfile)
return destfile
def find_linux_kernel(self, suite, flavour, arch):
if flavour == None:
rmad = run_cmd('rmadison', 'linux-image-2.6-%s' % (arch))
else:
rmad = run_cmd('rmadison', 'linux-image-2.6-%s-%s' % (flavour, arch))
version = ['0', '0','0', '0']
for line in rmad.splitlines():
sline = line.split('|')
#Linux XEN kernel is referred to in Debian by rmadison as:
#linux-image-2.6-xen-amd64 | 2.6.18+6etch3 | oldstable | amd64
#Specifically, etch is called 'oldstable' in the 3rd field, to get the suite you need
#excavate it from the 2nd field.
if sline[1].strip().count(suite) > 0:
#Fix for Debian handling of kernel version names
#It's x.y.z+w, not x.y.z.w
vt = sline[1].strip().split('.')
deb_vt = vt[2].split('+')
vt = [vt[0], vt[1], deb_vt[0], deb_vt[1]]
for i in range(4):
if int(vt[i]) > int(version[i]):
version = vt
break
if version[0] != '0':
return '%s.%s.%s-%s' % (version[0],version[1],version[2],version[3])
else:
return None
def create(self):
logging.info('Creating filesystem: %s, size: %d, dummy: %s' %
(self.mntpnt, self.size, repr(self.dummy)))
if not self.preallocated:
logging.info('Not preallocated, so we create it.')
if not self.filename:
if self.mntpnt:
self.filename = re.sub('[^\w\s/]', '',
self.mntpnt).strip().lower()
self.filename = re.sub('[\w/]', '_', self.filename)
if self.filename == '_':
self.filename = 'root'
elif self.type == TYPE_SWAP:
self.filename = 'swap'
else:
raise VMBuilderException('mntpnt not set')
self.filename = '%s/%s' % (self.vm.workdir, self.filename)
while os.path.exists('%s.img' % self.filename):
self.filename += '_'
self.filename += '.img'
logging.info(
'A name wasn\'t specified either, so we make one up: %s' %
self.filename)
run_cmd(qemu_img_path(), 'create', '-f', 'raw', self.filename,
'%dM' % self.size)
self.mkfs()
def convert(self, destdir, format):
"""
Convert the disk image
@type destdir: string
@param destdir: Target location of converted disk image
@type format: string
@param format: The target format (as understood by qemu-img or vdi)
@rtype: string
@return: the name of the converted image
"""
if self.preallocated:
# We don't convert preallocated disk images. That would be silly.
return self.filename
filename = os.path.basename(self.filename)
if '.' in filename:
filename = filename[:filename.rindex('.')]
destfile = '%s/%s.%s' % (destdir, filename, format)
logging.info('Converting %s to %s, format %s' % (self.filename, format, destfile))
if format == 'vdi':
run_cmd(vbox_manager_path(), 'convertfromraw', '-format', 'VDI', self.filename, destfile)
else:
run_cmd(qemu_img_path(), 'convert', '-O', format, self.filename, destfile)
os.unlink(self.filename)
self.filename = os.path.abspath(destfile)
self.format_type = format
return destfile
def install_bootloader_cleanup(self, chroot_dir):
self.context.cancel_cleanup(self.install_bootloader_cleanup)
tmpdir = '%s/tmp/vmbuilder-grub' % chroot_dir
for disk in os.listdir(tmpdir):
if disk != 'device.map':
run_cmd('umount', os.path.join(tmpdir, disk))
shutil.rmtree(tmpdir)
def unmap(self, ignore_fail=False):
"""
Destroy all mapping devices
Unsets L{Partition}s' and L{Filesystem}s' filename attribute
"""
# first sleep to give the loopback devices a chance to settle down
time.sleep(3)
tries = 0
max_tries = 3
while tries < max_tries:
try:
run_cmd('kpartx', '-d', self.filename, ignore_fail=False)
break
except:
pass
tries += 1
time.sleep(3)
if tries >= max_tries:
# try it one last time
logging.info("Could not unmap '%s' after '%d' attempts. Final attempt" % (self.filename, tries))
run_cmd('kpartx', '-d', self.filename, ignore_fail=ignore_fail)
for part in self.partitions:
part.set_filename(None)
def rinse(self):
# Work around bug in rinse: it doesn't set the rpm platform file,
# so yum uses os.uname to get $arch and tries to install packages
# for the wrong architecture
os.mkdir('%s/etc' % self.destdir, 0755)
os.mkdir('%s/etc/rpm' % self.destdir, 0755)
if self.vm.arch == 'amd64':
self.vm.install_file('/etc/rpm/platform', 'x86_64-redhat-linux')
else:
self.vm.install_file('/etc/rpm/platform', 'i686-redhat-linux')
# Let's select a mirror for installation
if not self.vm.install_mirror:
if self.vm.mirror:
self.vm.install_mirror = self.vm.mirror
else:
self.vm.install_mirror = 'http://mirror.bytemark.co.uk/centos'
# Create temporary rinse config file so we can force a mirror
# Of course, rinse will only use the mirror to download the
# initial packages itself, once it spawns yum in the chroot, yum
# uses the default config file.
(rinse_conf_handle, rinse_conf_name) = tempfile.mkstemp()
self.vm.add_clean_cb(lambda:os.remove(rinse_conf_name))
os.write(rinse_conf_handle, self.rinse_conf % (self.vm.suite, self.vm.install_mirror, self.vm.install_mirror))
self.vm.add_clean_cmd('umount', '%s/proc' % self.destdir, ignore_fail=True)
cmd = ['/usr/sbin/rinse', '--config', rinse_conf_name, '--arch', self.vm.arch, '--distribution', self.vm.suite, '--directory', self.destdir ]
run_cmd(*cmd)
def unmap(self, ignore_fail=False):
"""
Destroy all mapping devices
"""
run_cmd('kpartx', '-d', self.filename, ignore_fail=ignore_fail)
for part in self.partitions:
self.mapdev = None
def deploy(self):
if not getattr(self.vm, 'ec2', False):
return False
if self.context.ec2_bundle:
logging.info("Building EC2 bundle")
bundle_cmdline = ['ec2-bundle-image', '--image', self.context.filesystems[0].filename, '--cert', self.vm.ec2_cert, '--privatekey', self.vm.ec2_key, '--user', self.vm.ec2_user, '--prefix', self.vm.ec2_name, '-r', ['i386', 'x86_64'][self.vm.arch == 'amd64'], '-d', self.vm.workdir, '--kernel', self.vm.ec2_kernel, '--ramdisk', self.vm.ec2_ramdisk]
run_cmd(*bundle_cmdline)
manifest = '%s/%s.manifest.xml' % (self.context.workdir, self.vm.ec2_name)
if self.context.ec2_upload:
logging.info("Uploading EC2 bundle")
upload_cmdline = ['ec2-upload-bundle', '--retry', '--manifest', manifest, '--bucket', self.context.ec2_bucket, '--access-key', self.vm.ec2_access_key, '--secret-key', self.vm.ec2_secret_key]
run_cmd(*upload_cmdline)
if self.context.ec2_register:
from boto.ec2.connection import EC2Connection
conn = EC2Connection(self.context.ec2_access_key, self.vm.ec2_secret_key)
amiid = conn.register_image('%s/%s.manifest.xml' % (self.context.ec2_bucket, self.vm.ec2_name))
print 'Image registered as %s' % amiid
else:
self.context.result_files.append(manifest)
else:
self.context.result_files.append(self.vm.filesystems[0].filename)
return True
def install_grub(self):
self.run_in_target('apt-get', '--force-yes', '-y', 'install', 'grub')
run_cmd(
'rsync', '-a', '%s%s/%s/' %
(self.destdir, self.grubroot,
self.vm.arch == 'amd64' and 'x86_64-pc' or 'i386-pc'),
'%s/boot/grub/' % self.destdir)
def install_bootloader_cleanup(self, chroot_dir):
self.context.cancel_cleanup(self.install_bootloader_cleanup)
tmpdir = '%s/tmp/vmbuilder-grub' % chroot_dir
for disk in os.listdir(tmpdir):
if disk != 'device.map':
run_cmd('umount', os.path.join(tmpdir, disk))
shutil.rmtree(tmpdir)
def unmap(self, ignore_fail=False):
"""
Destroy all mapping devices
"""
# first sleep to give the loopback devices a chance to settle down
time.sleep(3)
tries = 0
max_tries = 3
while tries < max_tries:
try:
run_cmd('kpartx', '-d', self.filename, ignore_fail=False)
break
except:
pass
tries += 1
time.sleep(3)
if tries >= max_tries:
# try it one last time
logging.info(
"Could not unmount '%s' after '%d' attempts. Final attempt"
% (self.filename, tries))
run_cmd('kpartx', '-d', self.filename, ignore_fail=ignore_fail)
for part in self.partitions:
self.mapdev = None
def preflight_check(self):
"""While not all of these are strictly checks, their failure would inevitably
lead to failure, and since we can check them before we start setting up disk
and whatnot, we might as well go ahead an do this now."""
mysuite = self.get_setting("suite")
if not mysuite in self.suites:
raise VMBuilderUserError('Invalid suite. Valid suites are: %s' % ' '.join(self.suites))
modname = 'VMBuilder.plugins.centos.%s' % (mysuite.replace('-',''), )
mod = __import__(modname, fromlist=[mysuite.replace('-','')])
self.suite = getattr(mod, mysuite.replace('-','').capitalize())(self)
myarch = self.get_setting("arch")
if myarch not in self.valid_archs[self.host_arch] or \
not self.suite.check_arch_validity(myarch):
raise VMBuilderUserError('%s is not a valid architecture. Valid architectures are: %s' % (myarch,
' '.join(self.valid_archs[self.host_arch])))
#myhypervisor = self.get_setting('hypervisor')
#if myhypervisor.name == 'Xen':
# logging.info('Xen kernel default: linux-image-%s %s', self.suite.xen_kernel_flavour, self.xen_kernel_version())
self.virtio_net = self.use_virtio_net()
mylang = self.get_setting("lang")
if mylang:
try:
run_cmd('locale-gen', '%s' % mylang)
except VMBuilderException, e:
msg = "locale-gen does not recognize your locale '%s'" % mylang
raise VMBuilderUserError(msg)
def install_bootloader(self, chroot_dir, disks):
tmpdir = '/tmp/vmbuilder-grub'
os.makedirs('%s%s' % (chroot_dir, tmpdir))
self.add_clean_cb(self.install_bootloader_cleanup)
devmapfile = os.path.join(tmpdir, 'device.map')
devmap = open('%s%s' % (chroot_dir, devmapfile), 'w')
for (disk, id) in zip(disks, range(len(disks))):
new_filename = os.path.join(tmpdir, os.path.basename(disk.filename))
open('%s%s' % (chroot_dir, new_filename), 'w').close()
run_cmd('mount', '--bind', disk.filename, '%s%s' % (chroot_dir, new_filename))
devmap.write("(hd%d) %s\n" % (id, new_filename))
devmap.close()
#
# There are a couple of reasons why grub installation can fail:
#
# "Error 2: Bad file or directory type" can be caused by an ext3
# partition with 256 bit inodes and an older distro. See workaround
# in disk.py.
#
# "Error 18: Selected cylinder exceeds maximum supported by BIOS"
# can be caused by grub detecting a geometry that may not be
# compatible with an older BIOS. We work around this below by
# setting the geometry with bogus values:
#
self.run_in_target('grub', '--device-map=%s' % devmapfile, '--batch', stdin='''root (hd0,0)
geometry (hd0) 800 800 800
setup (hd0)
EOT''')
self.install_bootloader_cleanup()
def find_linux_kernel(self, suite, flavour, arch):
if flavour == None:
rmad = run_cmd('rmadison', 'linux-image-2.6-%s' % (arch))
else:
rmad = run_cmd('rmadison',
'linux-image-2.6-%s-%s' % (flavour, arch))
version = ['0', '0', '0', '0']
for line in rmad.splitlines():
sline = line.split('|')
#Linux XEN kernel is referred to in Debian by rmadison as:
#linux-image-2.6-xen-amd64 | 2.6.18+6etch3 | oldstable | amd64
#Specifically, etch is called 'oldstable' in the 3rd field, to get the suite you need
#excavate it from the 2nd field.
if sline[1].strip().count(suite) > 0:
#Fix for Debian handling of kernel version names
#It's x.y.z+w, not x.y.z.w
vt = sline[1].strip().split('.')
deb_vt = vt[2].split('+')
vt = [vt[0], vt[1], deb_vt[0], deb_vt[1]]
for i in range(4):
if int(vt[i]) > int(version[i]):
version = vt
break
if version[0] != '0':
return '%s.%s.%s-%s' % (version[0], version[1], version[2],
version[3])
else:
return None
def divert_file(self, path, add):
if add: action = "--add"
else: action = "--remove"
if not add:
os.remove('%s/%s' % (self.context.chroot_dir, path))
run_cmd('chroot', self.context.chroot_dir, 'dpkg-divert', '--local',
'--rename', action, path)
def mount(self, rootmnt):
if (self.type != TYPE_SWAP) and not self.dummy:
logging.debug('Mounting %s', self.mntpnt)
self.mntpath = '%s%s' % (rootmnt, self.mntpnt)
if not os.path.exists(self.mntpath):
os.makedirs(self.mntpath)
run_cmd('mount', '-o', 'loop', self.filename, self.mntpath)
self.vm.add_clean_cb(self.umount)
def mkfs(self):
if not self.dummy:
cmd = self.mkfs_fstype() + [self.filename]
run_cmd(*cmd)
if os.path.exists("/sbin/vol_id"):
self.uuid = run_cmd('vol_id', '--uuid', self.filename).rstrip()
elif os.path.exists("/sbin/blkid"):
self.uuid = run_cmd('blkid', '-sUUID', '-ovalue', self.filename).rstrip()
def mount(self):
if (self.type != TYPE_SWAP) and not self.dummy:
logging.debug('Mounting %s', self.mntpnt)
self.mntpath = '%s%s' % (self.vm.rootmnt, self.mntpnt)
if not os.path.exists(self.mntpath):
os.makedirs(self.mntpath)
run_cmd('mount', '-o', 'loop', self.filename, self.mntpath)
self.vm.add_clean_cb(self.umount)
def debootstrap(self):
cmd = ['/usr/sbin/debootstrap', '--arch=%s' % self.vm.arch]
if self.vm.variant:
cmd += ['--variant=%s' % self.vm.variant]
cmd += [self.vm.suite, self.destdir, self.debootstrap_mirror()]
kwargs = {'env': {'DEBIAN_FRONTEND': 'noninteractive'}}
if self.vm.proxy:
kwargs['env']['http_proxy'] = self.vm.proxy
run_cmd(*cmd, **kwargs)
def create(self, disk):
"""Adds partition to the disk image (does not mkfs or anything like that)"""
logging.info('Adding type %d partition to disk image: %s' % (self.type, disk.filename))
if self.begin == 0:
logging.info('Partition at beginning of disk - reserving first cylinder')
partition_start = "63s"
else:
partition_start = self.begin
run_cmd('parted', '--script', '--', disk.filename, 'mkpart', 'primary', self.parted_fstype(), partition_start, self.end)
def install_bootloader(self):
devmapfile = '%s/device.map' % self.vm.workdir
devmap = open(devmapfile, 'w')
for (disk, id) in zip(self.vm.disks, range(len(self.vm.disks))):
devmap.write("(hd%d) %s\n" % (id, disk.filename))
devmap.close()
run_cmd('grub', '--device-map=%s' % devmapfile, '--batch', stdin='''root (hd0,0)
setup (hd0)
EOT''')
def create(self, disk):
"""Adds partition to the disk image (does not mkfs or anything like that)"""
logging.info('Adding type %d partition to disk image: %s' % (self.type, disk.filename))
if self.begin == 0:
logging.info('Partition at beginning of disk - reserving first cylinder')
partition_start = "63s"
else:
partition_start = self.begin
run_cmd('parted', '--script', '--', disk.filename, 'mkpart', 'primary', self.parted_fstype(), partition_start, self.end)
def mount_dev_proc(self):
run_cmd('mount', '--bind', '/dev', '%s/dev' % self.context.chroot_dir)
self.context.add_clean_cb(self.unmount_dev)
run_cmd('mount', '--bind', '/dev/pts', '%s/dev/pts' % self.context.chroot_dir)
self.context.add_clean_cb(self.unmount_dev_pts)
self.run_in_target('mount', '-t', 'proc', 'proc', '/proc')
self.context.add_clean_cb(self.unmount_proc)
def mkfs(self):
if not self.dummy:
cmd = self.mkfs_fstype() + [self.filename]
run_cmd(*cmd)
if os.path.exists("/sbin/vol_id"):
self.uuid = run_cmd('vol_id', '--uuid', self.filename).rstrip()
elif os.path.exists("/sbin/blkid"):
self.uuid = run_cmd('blkid', '-sUUID', '-ovalue',
self.filename).rstrip()
def debootstrap(self):
cmd = ['/usr/sbin/debootstrap', '--arch=%s' % self.vm.arch]
if self.vm.variant:
cmd += ['--variant=%s' % self.vm.variant]
cmd += [self.vm.suite, self.destdir, self.debootstrap_mirror()]
kwargs = { 'env' : { 'DEBIAN_FRONTEND' : 'noninteractive' } }
if self.vm.proxy:
kwargs['env']['http_proxy'] = self.vm.proxy
run_cmd(*cmd, **kwargs)
def install_kernel(self, destdir):
run_cmd('chroot',
destdir,
'apt-get',
'--force-yes',
'-y',
'install',
self.kernel_name(),
env={'DEBIAN_FRONTEND': 'noninteractive'})
def mount_dev_proc(self):
run_cmd("mount", "--bind", "/dev", "%s/dev" % self.context.chroot_dir)
self.context.add_clean_cb(self.unmount_dev)
run_cmd("mount", "--bind", "/dev/pts", "%s/dev/pts" % self.context.chroot_dir)
self.context.add_clean_cb(self.unmount_dev_pts)
self.run_in_target("mount", "-t", "proc", "proc", "/proc")
self.context.add_clean_cb(self.unmount_proc)
def mount_dev_proc(self):
run_cmd('mount', '--bind', '/dev', '%s/dev' % self.context.chroot_dir)
self.context.add_clean_cb(self.unmount_dev)
run_cmd('mount', '--bind', '/dev/pts', '%s/dev/pts' % self.context.chroot_dir)
self.context.add_clean_cb(self.unmount_dev_pts)
self.run_in_target('mount', '-t', 'proc', 'proc', '/proc')
self.context.add_clean_cb(self.unmount_proc)
def mount_dev_proc(self):
run_cmd('mount', '--bind', '/dev', '%s/dev' % self.destdir)
self.vm.add_clean_cmd('umount', '%s/dev' % self.destdir, ignore_fail=True)
run_cmd('mount', '--bind', '/dev/pts', '%s/dev/pts' % self.destdir)
self.vm.add_clean_cmd('umount', '%s/dev/pts' % self.destdir, ignore_fail=True)
self.run_in_target('mount', '-t', 'proc', 'proc', '/proc')
self.vm.add_clean_cmd('umount', '%s/proc' % self.destdir, ignore_fail=True)
def post_install(self):
if self.vm.copy:
logging.info("Copying files specified by --copy in: %s" % self.vm.copy)
try:
for line in file(self.vm.copy):
pair = line.strip().split(' ')
util.run_cmd('cp', '-LpR', pair[0], '%s%s' % (self.vm.installdir, pair[1]))
except IOError, (errno, strerror):
raise VMBuilderUserError("%s executing --copy directives: %s" % (errno, strerror))
def install_grub(self, chroot_dir):
self.install_from_template("/etc/kernel-img.conf", "kernelimg", {"updategrub": self.updategrub})
arch = self.context.get_setting("arch")
self.run_in_target("apt-get", "--force-yes", "-y", "install", "grub", env={"DEBIAN_FRONTEND": "noninteractive"})
run_cmd(
"rsync",
"-a",
"%s%s/%s/" % (chroot_dir, self.grubroot, arch == "amd64" and "x86_64-pc" or "i386-pc"),
"%s/boot/grub/" % chroot_dir,
)
def debootstrap_mirror(self):
if self.vm.iso:
isodir = tempfile.mkdtemp()
self.vm.add_clean_cb(lambda:os.rmdir(isodir))
run_cmd('mount', '-o', 'loop', '-t', 'iso9660', self.vm.iso, isodir)
self.vm.add_clean_cmd('umount', isodir)
self.iso_mounted = True
return 'file://%s' % isodir
else:
return self.install_mirrors()[0]
def create(self):
"""
Creates the disk image (if it doesn't already exist).
Once this method returns succesfully, L{filename} can be
expected to points to point to whatever holds the virtual disk
(be it a file, partition, logical volume, etc.).
"""
if not os.path.exists(self.filename):
logging.info('Creating disk image: "%s" of size: %dMB' % (self.filename, self.size))
run_cmd(qemu_img_path(), 'create', '-f', 'raw', self.filename, '%dM' % self.size)
def install_kernel(self, destdir):
run_cmd(
"chroot",
destdir,
"apt-get",
"--force-yes",
"-y",
"install",
self.kernel_name(),
env={"DEBIAN_FRONTEND": "noninteractive"},
)
def create(self):
"""
Creates the disk image (if it doesn't already exist).
Once this method returns succesfully, L{filename} can be
expected to points to point to whatever holds the virtual disk
(be it a file, partition, logical volume, etc.).
"""
if not os.path.exists(self.filename):
logging.info('Creating disk image: "%s" of size: %dMB' % (self.filename, self.size))
run_cmd(qemu_img_path(), 'create', '-f', 'raw', self.filename, '%dM' % self.size)
def convert(self, filesystems, destdir):
destimages = []
for filesystem in filesystems:
if not filesystem.preallocated:
destfile = '%s/%s' % (destdir,
os.path.basename(filesystem.filename))
logging.info('Moving %s to %s' %
(filesystem.filename, destfile))
run_cmd('cp', '--sparse=always', filesystem.filename, destfile)
self.call_hooks('fix_ownership', destfile)
os.unlink(filesystem.filename)
filesystem.filename = os.path.abspath(destfile)
destimages.append(destfile)
if not self.context.get_setting('xen-kernel'):
self.context.xen_kernel = self.context.distro.xen_kernel_path()
if not self.context.get_setting('xen-ramdisk'):
self.context.xen_ramdisk = self.context.distro.xen_ramdisk_path()
xenconf = '%s/xen.conf' % destdir
fp = open(xenconf, 'w')
fp.write("""
# Configuration file for the Xen instance %s, created
# by VMBuilder
kernel = '%s'
ramdisk = '%s'
memory = %d
root = '/dev/xvda1 ro'
disk = [
%s
]
name = '%s'
dhcp = 'dhcp'
vif = ['']
on_poweroff = 'destroy'
on_reboot = 'restart'
on_crash = 'restart'
extra = 'xencons=tty console=tty1 console=hvc0'
""" % (self.context.distro.get_setting('hostname'),
self.context.get_setting('xen-kernel'),
self.context.get_setting('xen-ramdisk'),
self.context.get_setting('mem'), ',\n'.join([
"'tap:aio:%s,xvda%d,w'" % (os.path.abspath(img), id + 1)
for (img, id) in zip(destimages, range(len(destimages)))
]), self.context.distro.get_setting('hostname')))
fp.close()
self.call_hooks('fix_ownership', xenconf)
def debootstrap_mirror(self):
if self.vm.iso:
isodir = tempfile.mkdtemp()
self.vm.add_clean_cb(lambda: os.rmdir(isodir))
run_cmd('mount', '-o', 'loop', '-t', 'iso9660', self.vm.iso,
isodir)
self.vm.add_clean_cmd('umount', isodir)
self.iso_mounted = True
return 'file://%s' % isodir
else:
return self.install_mirrors()[0]
def debootstrap_mirror(self):
iso = self.context.get_setting("iso")
if iso:
isodir = tempfile.mkdtemp()
self.context.add_clean_cb(lambda: os.rmdir(isodir))
run_cmd("mount", "-o", "loop", "-t", "iso9660", iso, isodir)
self.context.add_clean_cmd("umount", isodir)
self.iso_mounted = True
return "file://%s" % isodir
else:
return self.install_mirrors()[0]
def create(self, directory):
"""
Creates the disk image (unless preallocated), partitions it, creates the partition mapping devices and mkfs's the partitions
@type directory: string
@param directory: If set, the disk image is created in this directory
"""
if not self.preallocated:
if directory:
self.filename = '%s/%s' % (directory, self.filename)
logging.info('Creating disk image: %s' % self.filename)
run_cmd(qemu_img_path(), 'create', '-f', 'raw', self.filename,
'%dM' % self.size)
os.chmod(self.filename, stat.S_IRUSR | stat.S_IWUSR)
# From here, we assume that self.filename refers to whatever holds the disk image,
# be it a file, a partition, logical volume, actual disk..
logging.info('Adding partition table to disk image: %s' %
self.filename)
run_cmd('parted', '--script', self.filename, 'mklabel', 'msdos')
# Partition the disk
for part in self.partitions:
part.create(self)
logging.info(
'Creating loop devices corresponding to the created partitions')
self.vm.add_clean_cb(lambda: self.unmap(ignore_fail=True))
kpartx_output = run_cmd('kpartx', '-av', self.filename)
parts = []
for line in kpartx_output.split('\n'):
if line == "" or line.startswith("gpt:") or line.startswith(
"dos:"):
continue
if line.startswith("add"):
parts.append(line)
continue
logging.error('Skipping unknown line in kpartx output (%s)' % line)
mapdevs = []
for line in parts:
mapdevs.append(line.split(' ')[2])
for (part, mapdev) in zip(self.partitions, mapdevs):
part.mapdev = '/dev/mapper/%s' % mapdev
# At this point, all partitions are created and their mapping device has been
# created and set as .mapdev
# Adds a filesystem to the partition
logging.info("Creating file systems")
for part in self.partitions:
part.mkfs()
def install_bootloader(self):
devmapfile = '%s/device.map' % self.vm.workdir
devmap = open(devmapfile, 'w')
for (disk, id) in zip(self.vm.disks, range(len(self.vm.disks))):
devmap.write("(hd%d) %s\n" % (id, disk.filename))
devmap.close()
run_cmd('grub',
'--device-map=%s' % devmapfile,
'--batch',
stdin='''root (hd0,0)
setup (hd0)
EOT''')
def create(self, directory):
"""
Creates the disk image (unless preallocated), partitions it, creates the partition mapping devices and mkfs's the partitions
@type directory: string
@param directory: If set, the disk image is created in this directory
"""
if not self.preallocated:
if directory:
self.filename = '%s/%s' % (directory, self.filename)
logging.info('Creating disk image: %s' % self.filename)
qemu_img_output = run_cmd(qemu_img_path(), 'create', '-f', 'raw', self.filename, '%dM' % self.size)
if not os.path.exists(self.filename):
logging.info("Problem while creating raw image: %s" % qemu_img_output)
raise Exception("Problem while creating raw image: %s" % qemu_img_output)
# From here, we assume that self.filename refers to whatever holds the disk image,
# be it a file, a partition, logical volume, actual disk..
logging.info('Adding partition table to disk image: %s' % self.filename)
run_cmd('parted', '--script', self.filename, 'mklabel', 'msdos')
# Partition the disk
for part in self.partitions:
part.create(self)
logging.info('Creating loop devices corresponding to the created partitions')
self.vm.add_clean_cb(lambda : self.unmap(ignore_fail=True))
kpartx_output = run_cmd('kpartx', '-av', self.filename)
parts = []
for line in kpartx_output.split('\n'):
if line == "" or line.startswith("gpt:") or line.startswith("dos:"):
continue
if line.startswith("add"):
parts.append(line)
continue
logging.error('Skipping unknown line in kpartx output (%s)' % line)
mapdevs = []
for line in parts:
mapdevs.append(line.split(' ')[2])
for (part, mapdev) in zip(self.partitions, mapdevs):
part.mapdev = '/dev/mapper/%s' % mapdev
# At this point, all partitions are created and their mapping device has been
# created and set as .mapdev
# Adds a filesystem to the partition
logging.info("Creating file systems")
for part in self.partitions:
part.mkfs()
def finalize(self):
destimages = []
for filesystem in self.vm.filesystems:
if not filesystem.preallocated:
destfile = '%s/%s' % (self.vm.destdir,
os.path.basename(filesystem.filename))
logging.info('Moving %s to %s' %
(filesystem.filename, destfile))
self.vm.result_files.append(destfile)
run_cmd('cp', '--sparse=always', filesystem.filename, destfile)
os.unlink(filesystem.filename)
filesystem.filename = os.path.abspath(destfile)
destimages.append(destfile)
if not self.vm.xen_kernel:
self.vm.xen_kernel = self.vm.distro.xen_kernel_path()
if not self.vm.xen_ramdisk:
self.vm.xen_ramdisk = self.vm.distro.xen_ramdisk_path()
xenconf = '%s/xen.conf' % self.vm.destdir
fp = open(xenconf, 'w')
fp.write("""
# Configuration file for the Xen instance %s, created
# by VMBuilder
kernel = '%s'
ramdisk = '%s'
memory = %d
root = '/dev/xvda1 ro'
disk = [
%s
]
name = '%s'
dhcp = 'dhcp'
vif = ['']
on_poweroff = 'destroy'
on_reboot = 'restart'
on_crash = 'restart'
extra = 'xencons=tty console=tty1 console=hvc0'
""" % (self.vm.name, self.vm.xen_kernel, self.vm.xen_ramdisk, self.vm.mem,
',\n'.join([
"'tap:aio:%s,xvda%d,w'" % (os.path.abspath(img), id + 1)
for (img, id) in zip(destimages, range(len(destimages)))
]), self.vm.name))
fp.close()
self.vm.result_files.append(xenconf)
def convert(self, filesystems, destdir):
destimages = []
for filesystem in filesystems:
if not filesystem.preallocated:
destfile = '%s/%s' % (destdir, os.path.basename(filesystem.filename))
logging.info('Moving %s to %s' % (filesystem.filename, destfile))
run_cmd('cp', '--sparse=always', filesystem.filename, destfile)
self.call_hooks('fix_ownership', destfile)
os.unlink(filesystem.filename)
filesystem.filename = os.path.abspath(destfile)
destimages.append(destfile)
if not self.context.get_setting('xen-kernel'):
self.context.xen_kernel = self.context.distro.xen_kernel_path()
if not self.context.get_setting('xen-ramdisk'):
self.context.xen_ramdisk = self.context.distro.xen_ramdisk_path()
xenconf = '%s/xen.conf' % destdir
fp = open(xenconf, 'w')
fp.write("""
# Configuration file for the Xen instance %s, created
# by VMBuilder
kernel = '%s'
ramdisk = '%s'
memory = %d
root = '/dev/xvda1 ro'
disk = [
%s
]
name = '%s'
dhcp = 'dhcp'
vif = ['']
on_poweroff = 'destroy'
on_reboot = 'restart'
on_crash = 'restart'
extra = 'xencons=tty console=tty1 console=hvc0'
""" % (self.context.distro.get_setting('hostname'),
self.context.get_setting('xen-kernel'),
self.context.get_setting('xen-ramdisk'),
self.context.get_setting('mem'),
',\n'.join(["'tap:aio:%s,xvda%d,w'" % (os.path.abspath(img), id+1) for (img, id) in zip(destimages, range(len(destimages)))]),
self.context.distro.get_setting('hostname')))
fp.close()
self.call_hooks('fix_ownership', xenconf)
def partition(self):
"""
Partitions the disk image. First adds a partition table and then
adds the individual partitions.
Should only be called once and only after you've added all partitions.
"""
logging.info('Adding partition table to disk image: %s' % self.filename)
run_cmd('parted', '--script', self.filename, 'mklabel', 'msdos')
# Partition the disk
for part in self.partitions:
part.create(self)
def finalize(self):
destimages = []
for filesystem in self.vm.filesystems:
if not filesystem.preallocated:
destfile = '%s/%s' % (self.vm.destdir, os.path.basename(filesystem.filename))
logging.info('Moving %s to %s' % (filesystem.filename, destfile))
self.vm.result_files.append(destfile)
run_cmd('cp', '--sparse=always', filesystem.filename, destfile)
os.unlink(filesystem.filename)
filesystem.filename = os.path.abspath(destfile)
destimages.append(destfile)
if not self.vm.xen_kernel:
self.vm.xen_kernel = self.vm.distro.xen_kernel_path()
if not self.vm.xen_ramdisk:
self.vm.xen_ramdisk = self.vm.distro.xen_ramdisk_path()
xenconf = '%s/xen.conf' % self.vm.destdir
fp = open(xenconf, 'w')
fp.write("""
# Configuration file for the Xen instance %s, created
# by VMBuilder
kernel = '%s'
ramdisk = '%s'
memory = %d
root = '/dev/xvda1 ro'
disk = [
%s
]
name = '%s'
dhcp = 'dhcp'
vif = ['']
on_poweroff = 'destroy'
on_reboot = 'restart'
on_crash = 'restart'
extra = 'xencons=tty console=tty1 console=hvc0'
""" % (self.vm.name,
self.vm.xen_kernel,
self.vm.xen_ramdisk,
self.vm.mem,
',\n'.join(["'tap:aio:%s,xvda%d,w'" % (os.path.abspath(img), id+1) for (img, id) in zip(destimages, range(len(destimages)))]),
self.vm.name))
fp.close()
self.vm.result_files.append(xenconf)
def partition(self):
"""
Partitions the disk image. First adds a partition table and then
adds the individual partitions.
Should only be called once and only after you've added all partitions.
"""
logging.info('Adding partition table to disk image: %s' % self.filename)
run_cmd('parted', '--script', self.filename, 'mklabel', 'msdos')
# Partition the disk
for part in self.partitions:
part.create(self)
def install_grub(self, chroot_dir):
self.install_from_template('/etc/kernel-img.conf', 'kernelimg',
{'updategrub': self.updategrub})
arch = self.context.get_setting('arch')
self.run_in_target('apt-get',
'--force-yes',
'-y',
'install',
'grub',
env={'DEBIAN_FRONTEND': 'noninteractive'})
run_cmd(
'rsync', '-a',
'%s%s/%s/' % (chroot_dir, self.grubroot,
arch == 'amd64' and 'x86_64-pc' or 'i386-pc'),
'%s/boot/grub/' % chroot_dir)
def mount_dev_proc(self):
run_cmd('mount', '--bind', '/dev', '%s/dev' % self.destdir)
self.vm.add_clean_cmd('umount',
'%s/dev' % self.destdir,
ignore_fail=True)
run_cmd('mount', '--bind', '/dev/pts', '%s/dev/pts' % self.destdir)
self.vm.add_clean_cmd('umount',
'%s/dev/pts' % self.destdir,
ignore_fail=True)
self.run_in_target('mount', '-t', 'proc', 'proc', '/proc')
self.vm.add_clean_cmd('umount',
'%s/proc' % self.destdir,
ignore_fail=True)
def preflight_check(self):
if not getattr(self.vm, 'ec2', False):
return True
if not self.context.hypervisor.name == 'Xen':
raise VMBuilderUserError('When building for EC2 you must use the xen hypervisor.')
if self.context.ec2_bundle:
try:
run_cmd('ec2-ami-tools-version')
except VMBuilderException, e:
raise VMBuilderUserError('You need to have the Amazon EC2 AMI tools installed')
if not self.context.ec2_name:
raise VMBuilderUserError('When building for EC2 you must supply the name for the image.')
if not self.context.ec2_cert:
if "EC2_CERT" in os.environ:
self.context.ec2_cert = os.environ["EC2_CERT"]
else:
raise VMBuilderUserError('When building for EC2 you must provide your PEM encoded public key certificate')
if not self.context.ec2_key:
if "EC2_PRIVATE_KEY" in os.environ:
self.context.ec2_key = os.environ["EC2_PRIVATE_KEY"]
else:
raise VMBuilderUserError('When building for EC2 you must provide your PEM encoded private key file')
if not self.context.ec2_user:
raise VMBuilderUserError('When building for EC2 you must provide your EC2 user ID (your AWS account number, not your AWS access key ID)')
if not self.context.ec2_kernel:
self.context.ec2_kernel = self.vm.distro.get_ec2_kernel()
logging.debug('%s - to be used for AKI.' %(self.context.ec2_kernel))
if not self.context.ec2_ramdisk:
self.context.ec2_ramdisk = self.vm.distro.ec2_ramdisk_id()
logging.debug('%s - to be use for the ARI.' %(self.context.ec2_ramdisk))
if self.context.ec2_upload:
if not self.context.ec2_bucket:
raise VMBuilderUserError('When building for EC2 you must provide an S3 bucket to hold the AMI')
if not self.context.ec2_access_key:
raise VMBuilderUserError('When building for EC2 you must provide your AWS access key ID.')
if not self.context.ec2_secret_key:
raise VMBuilderUserError('When building for EC2 you must provide your AWS secret access key.')
def xen_kernel_version(self):
if self.suite.xen_kernel_flavour:
# if this is ec2, do not call rmadison.
# this could be replaced with a method to get most recent
# stable kernel, but really, this is not used at all for ec2
if hasattr(self.context, 'ec2') and self.context.ec2:
logging.debug("selecting ec2 kernel")
self.xen_kernel = "2.6.ec2-kernel"
return self.xen_kernel
if not self.xen_kernel:
rmad = run_cmd('rmadison', 'linux-image-%s' % self.suite.xen_kernel_flavour)
version = ['0', '0','0', '0']
for line in rmad.splitlines():
sline = line.split('|')
if sline[2].strip().startswith(self.context.get_setting('suite')):
vt = sline[1].strip().split('.')
for i in range(4):
if int(vt[i]) > int(version[i]):
version = vt
break
if version[0] == '0':
raise VMBuilderException('Something is wrong, no valid xen kernel for the suite %s found by rmadison' % self.context.suite)
self.xen_kernel = '%s.%s.%s-%s' % (version[0],version[1],version[2],version[3])
return self.xen_kernel
else:
raise VMBuilderUserError('There is no valid xen kernel for the suite selected.')
def map_partitions(self):
"""
Create loop devices corresponding to the partitions.
Once this has returned succesfully, each partition's map device
is set as its L{filename<Disk.Partition.filename>} attribute.
Call this after L{partition}.
"""
logging.info('Creating loop devices corresponding to the created partitions')
self.vm.add_clean_cb(lambda : self.unmap(ignore_fail=True))
kpartx_output = run_cmd('kpartx', '-asv', self.filename)
parts = []
for line in kpartx_output.split('\n'):
if line == "" or line.startswith("gpt:") or line.startswith("dos:"):
continue
if line.startswith("add"):
parts.append(line)
continue
logging.error('Skipping unknown line in kpartx output (%s)' % line)
mapdevs = []
for line in parts:
mapdevs.append(line.split(' ')[2])
for (part, mapdev) in zip(self.partitions, mapdevs):
part.set_filename('/dev/mapper/%s' % mapdev)
def test_partition_table_empty(self):
from VMBuilder.util import run_cmd
file_output = run_cmd('file', self.tmpfile)
self.assertEqual('%s: data' % self.tmpfile, file_output.strip())
self.disk.partition()
file_output = run_cmd('file', self.tmpfile)
self.assertEqual('%s: x86 boot sector, code offset 0xb8' % self.tmpfile, file_output.strip())
file_output = run_cmd('parted', '--script', self.tmpfile, 'print')
self.assertEqual('''Model: (file)
Disk %s: 1074MB
Sector size (logical/physical): 512B/512B
Partition Table: msdos
Number Start End Size Type File system Flags''' % self.tmpfile, file_output.strip())
def map_partitions(self):
"""
Create loop devices corresponding to the partitions.
Once this has returned succesfully, each partition's map device
is set as its L{filename<Disk.Partition.filename>} attribute.
Call this after L{partition}.
"""
logging.info('Creating loop devices corresponding to the created partitions')
self.vm.add_clean_cb(lambda : self.unmap(ignore_fail=True))
kpartx_output = run_cmd('kpartx', '-av', self.filename)
parts = []
for line in kpartx_output.split('\n'):
if line == "" or line.startswith("gpt:") or line.startswith("dos:"):
continue
if line.startswith("add"):
parts.append(line)
continue
logging.error('Skipping unknown line in kpartx output (%s)' % line)
mapdevs = []
for line in parts:
mapdevs.append(line.split(' ')[2])
for (part, mapdev) in zip(self.partitions, mapdevs):
part.set_filename('/dev/mapper/%s' % mapdev)
def xen_kernel_version(self):
if self.suite.xen_kernel_flavour:
if not self.xen_kernel:
rmad = run_cmd(
'rmadison',
'linux-image-%s' % self.suite.xen_kernel_flavour)
version = ['0', '0', '0', '0']
for line in rmad.splitlines():
sline = line.split('|')
if sline[2].strip().startswith(self.vm.suite):
vt = sline[1].strip().split('.')
for i in range(4):
if int(vt[i]) > int(version[i]):
version = vt
break
if version[0] == '0':
raise VMBuilderException(
'Something is wrong, no valid xen kernel for the suite %s found by rmadison'
% self.vm.suite)
self.xen_kernel = '%s.%s.%s-%s' % (version[0], version[1],
version[2], version[3])
return self.xen_kernel
else:
raise VMBuilderUserError(
'There is no valid xen kernel for the suite selected.')
def unmount_volatile(self):
for mntpnt in glob.glob('%s/lib/modules/*/volatile' % self.destdir):
logging.debug("Unmounting %s" % mntpnt)
run_cmd('umount', mntpnt)
# Finally unmounting /dev and /proc. Maybe not very clean to do that here.
run_cmd('umount', '%s/dev/pts' % self.destdir)
run_cmd('umount', '%s/dev' % self.destdir)
run_cmd('umount', '%s/proc' % self.destdir)
def install_os(self):
self.nics = [self.NIC()]
self.call_hooks('preflight_check')
self.call_hooks('configure_networking', self.nics)
self.call_hooks('configure_mounting', self.disks, self.filesystems)
self.chroot_dir = tmpdir()
self.call_hooks('mount_partitions', self.chroot_dir)
run_cmd('rsync', '-aHA', '%s/' % self.distro.chroot_dir,
self.chroot_dir)
self.distro.set_chroot_dir(self.chroot_dir)
if self.needs_bootloader:
self.call_hooks('install_bootloader', self.chroot_dir, self.disks)
self.call_hooks('install_kernel', self.chroot_dir)
self.distro.call_hooks('post_install')
self.call_hooks('unmount_partitions')
os.rmdir(self.chroot_dir)
def post_install(self):
copy = self.context.get_setting('copy')
if copy:
logging.info("Copying files specified by copy in: %s" % copy)
try:
for line in file(copy):
pair = line.strip().split(' ')
if len(pair) < 2: # skip blank and incomplete lines
continue
directory = '%s%s' % (self.context.chroot_dir,
os.path.dirname(pair[1]))
if not os.path.exists(directory):
os.makedirs(directory)
util.run_cmd('cp', '-LpR', pair[0],
'%s%s' % (self.context.chroot_dir, pair[1]))
except IOError, (errno, strerror):
raise VMBuilderUserError("%s executing copy directives: %s" %
(errno, strerror))
