def run(cls, info):
vagrantfile_source = os.path.join(assets, 'Vagrantfile')
vagrantfile = os.path.join(info.vagrant['folder'], 'Vagrantfile')
shutil.copy(vagrantfile_source, vagrantfile)
import random
mac_address = '080027{mac:06X}'.format(mac=random.randrange(16**6))
from common.tools import sed_i
sed_i(vagrantfile, '\\[MAC_ADDRESS\\]', mac_address)
metadata_source = os.path.join(assets, 'metadata.json')
metadata = os.path.join(info.vagrant['folder'], 'metadata.json')
shutil.copy(metadata_source, metadata)
from common.tools import log_check_call
disk_name = 'box-disk1.{ext}'.format(ext=info.volume.extension)
disk_link = os.path.join(info.vagrant['folder'], disk_name)
log_check_call(['ln', '-s', info.volume.image_path, disk_link])
ovf_path = os.path.join(info.vagrant['folder'], 'box.ovf')
cls.write_ovf(info, ovf_path, mac_address, disk_name)
box_files = os.listdir(info.vagrant['folder'])
log_check_call([
'tar', '--create', '--gzip', '--dereference', '--file',
info.vagrant['box_path'], '--directory', info.vagrant['folder']
] + box_files)
import logging
logging.getLogger(__name__).info(
'The vagrant box has been placed at {box_path}'.format(
box_path=info.vagrant['box_path']))
def run(cls, info):
ssh_dir = os.path.join(info.root, "home/vagrant/.ssh")
os.mkdir(ssh_dir)
authorized_keys_source_path = os.path.join(assets, "authorized_keys")
with open(authorized_keys_source_path, "r") as authorized_keys_source:
insecure_public_key = authorized_keys_source.read()
authorized_keys_path = os.path.join(ssh_dir, "authorized_keys")
with open(authorized_keys_path, "a") as authorized_keys:
authorized_keys.write(insecure_public_key)
import stat
os.chmod(ssh_dir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
os.chmod(authorized_keys_path, stat.S_IRUSR | stat.S_IWUSR)
# We can't do this directly with python, since getpwnam gets its info from the host
from common.tools import log_check_call
log_check_call(
[
"chroot",
info.root,
"chown",
"vagrant:vagrant",
"/home/vagrant/.ssh",
"/home/vagrant/.ssh/authorized_keys",
]
)
def install_remote(cls, info, remote_packages):
import os
from common.tools import log_check_call
from subprocess import CalledProcessError
try:
env = os.environ.copy()
env['DEBIAN_FRONTEND'] = 'noninteractive'
log_check_call([
'chroot', info.root, 'apt-get', 'install',
'--no-install-recommends', '--assume-yes'
] + map(str, remote_packages),
env=env)
except CalledProcessError as e:
import logging
disk_stat = os.statvfs(info.root)
root_free_mb = disk_stat.f_bsize * disk_stat.f_bavail / 1024 / 1024
disk_stat = os.statvfs(os.path.join(info.root, 'boot'))
boot_free_mb = disk_stat.f_bsize * disk_stat.f_bavail / 1024 / 1024
free_mb = min(root_free_mb, boot_free_mb)
if free_mb < 50:
msg = ('apt exited with a non-zero status, '
'this may be because\nthe image volume is '
'running out of disk space ({free}MB left)').format(
free=free_mb)
logging.getLogger(__name__).warn(msg)
else:
if e.returncode == 100:
msg = (
'apt exited with status code 100. '
'This can sometimes occur when package retrieval times out or a package extraction failed. '
'apt might succeed if you try bootstrapping again.')
logging.getLogger(__name__).warn(msg)
raise e
def run(cls, info): executable, options, arguments = get_bootstrap_args(info) if hasattr(info, 'tarball'): options.extend(['--unpack-tarball=' + info.tarball]) from common.tools import log_check_call log_check_call(executable + options + arguments)
def install_remote(cls, info, remote_packages):
import os
from common.tools import log_check_call
from subprocess import CalledProcessError
try:
env = os.environ.copy()
env['DEBIAN_FRONTEND'] = 'noninteractive'
log_check_call(['chroot', info.root,
'apt-get', 'install',
'--no-install-recommends',
'--assume-yes']
+ map(str, remote_packages),
env=env)
except CalledProcessError as e:
import logging
disk_stat = os.statvfs(info.root)
root_free_mb = disk_stat.f_bsize * disk_stat.f_bavail / 1024 / 1024
disk_stat = os.statvfs(os.path.join(info.root, 'boot'))
boot_free_mb = disk_stat.f_bsize * disk_stat.f_bavail / 1024 / 1024
free_mb = min(root_free_mb, boot_free_mb)
if free_mb < 50:
msg = ('apt exited with a non-zero status, '
'this may be because\nthe image volume is '
'running out of disk space ({free}MB left)').format(free=free_mb)
logging.getLogger(__name__).warn(msg)
else:
if e.returncode == 100:
msg = ('apt exited with status code 100. '
'This can sometimes occur when package retrieval times out or a package extraction failed. '
'apt might succeed if you try bootstrapping again.')
logging.getLogger(__name__).warn(msg)
raise e
def run(cls, info):
from common.tools import log_check_call
for raw_command in info.manifest.plugins['image_commands']['commands']:
command = map(
lambda part: part.format(root=info.root, **info.manifest_vars),
raw_command)
log_check_call(command)
def install_local(cls, info, local_packages):
from shutil import copy
from common.tools import log_check_call
import os
absolute_package_paths = []
chrooted_package_paths = []
for package_src in local_packages:
pkg_name = os.path.basename(package_src.path)
package_rel_dst = os.path.join('tmp', pkg_name)
package_dst = os.path.join(info.root, package_rel_dst)
copy(package_src.path, package_dst)
absolute_package_paths.append(package_dst)
package_path = os.path.join('/', package_rel_dst)
chrooted_package_paths.append(package_path)
env = os.environ.copy()
env['DEBIAN_FRONTEND'] = 'noninteractive'
log_check_call(['chroot', info.root,
'dpkg', '--install']
+ chrooted_package_paths,
env=env)
for path in absolute_package_paths:
os.remove(path)
def _before_map(self, event):
volume = event.volume
try:
mappings = log_check_call(['/sbin/kpartx', '-l', volume.device_path])
import re
regexp = re.compile('^(?P<name>.+[^\d](?P<p_idx>\d+)) : '
'(?P<start_blk>\d) (?P<num_blks>\d+) '
'{device_path} (?P<blk_offset>\d+)$'
.format(device_path=volume.device_path))
log_check_call(['/sbin/kpartx', '-a', volume.device_path])
import os.path
for mapping in mappings:
match = regexp.match(mapping)
if match is None:
raise PartitionError('Unable to parse kpartx output: {line}'.format(line=mapping))
partition_path = os.path.join('/dev/mapper', match.group('name'))
p_idx = int(match.group('p_idx')) - 1
self.partitions[p_idx].map(partition_path)
for idx, partition in enumerate(self.partitions):
if partition.fsm.current not in ['mapped', 'formatted']:
raise PartitionError('kpartx did not map partition #{idx}'.format(idx=idx + 1))
except PartitionError as e:
for partition in self.partitions:
if not partition.fsm.can('unmap'):
partition.unmap()
log_check_call(['/sbin/kpartx', '-d', volume.device_path])
raise e
def mount(self, prefix): mount_dir = os.path.join(prefix, self.destination) if isinstance(self.source, AbstractPartition): self.source.mount(destination=mount_dir) else: log_check_call(['/bin/mount'] + self.opts + [self.source, mount_dir]) self.mount_dir = mount_dir
def run(cls, info):
vagrantfile_source = os.path.join(assets, 'Vagrantfile')
vagrantfile = os.path.join(info.vagrant['folder'], 'Vagrantfile')
shutil.copy(vagrantfile_source, vagrantfile)
import random
mac_address = '080027{mac:06X}'.format(mac=random.randrange(16 ** 6))
from common.tools import sed_i
sed_i(vagrantfile, '\\[MAC_ADDRESS\\]', mac_address)
metadata_source = os.path.join(assets, 'metadata.json')
metadata = os.path.join(info.vagrant['folder'], 'metadata.json')
shutil.copy(metadata_source, metadata)
from common.tools import log_check_call
disk_name = 'box-disk1.{ext}'.format(ext=info.volume.extension)
disk_link = os.path.join(info.vagrant['folder'], disk_name)
log_check_call(['ln', '-s', info.volume.image_path, disk_link])
ovf_path = os.path.join(info.vagrant['folder'], 'box.ovf')
cls.write_ovf(info, ovf_path, mac_address, disk_name)
box_files = os.listdir(info.vagrant['folder'])
log_check_call(['tar', '--create', '--gzip', '--dereference',
'--file', info.vagrant['box_path'],
'--directory', info.vagrant['folder']]
+ box_files
)
import logging
logging.getLogger(__name__).info('The vagrant box has been placed at {box_path}'
.format(box_path=info.vagrant['box_path']))
def run(cls, info):
executable, options, arguments = get_bootstrap_args(info)
if hasattr(info, 'tarball'):
options.extend(['--unpack-tarball=' + info.tarball])
from common.tools import log_check_call
log_check_call(executable + options + arguments)
def _before_link_dm_node(self, e):
"""Links the volume using the device mapper
This allows us to create a 'window' into the volume that acts like a volum in itself.
Mainly it is used to fool grub into thinking that it is working with a real volume,
rather than a loopback device or a network block device.
Args:
e (_e_obj): Event object containing arguments to create()
Arguments are:
logical_start_sector (int): The sector the volume should start at in the new volume
start_sector (int): The offset at which the volume should begin to be mapped in the new volume
sectors (int): The number of sectors that should be mapped
Read more at: http://manpages.debian.org/cgi-bin/man.cgi?query=dmsetup&apropos=0&sektion=0&manpath=Debian+7.0+wheezy&format=html&locale=en
Raises:
VolumeError
"""
import os.path
from common.fs import get_partitions
# Fetch information from /proc/partitions
proc_partitions = get_partitions()
device_name = os.path.basename(self.device_path)
device_partition = proc_partitions[device_name]
# The sector the volume should start at in the new volume
logical_start_sector = getattr(e, 'logical_start_sector', 0)
# The offset at which the volume should begin to be mapped in the new volume
start_sector = getattr(e, 'start_sector', 0)
# The number of sectors that should be mapped
sectors = getattr(e, 'sectors', int(self.size / 512) - start_sector)
# This is the table we send to dmsetup, so that it may create a decie mapping for us.
table = ('{log_start_sec} {sectors} linear {major}:{minor} {start_sec}'
.format(log_start_sec=logical_start_sector,
sectors=sectors,
major=device_partition['major'],
minor=device_partition['minor'],
start_sec=start_sector))
import string
import os.path
# Figure out the device letter and path
for letter in string.ascii_lowercase:
dev_name = 'vd' + letter
dev_path = os.path.join('/dev/mapper', dev_name)
if not os.path.exists(dev_path):
self.dm_node_name = dev_name
self.dm_node_path = dev_path
break
if not hasattr(self, 'dm_node_name'):
raise VolumeError('Unable to find a free block device path for mounting the bootstrap volume')
# Create the device mapping
log_check_call(['dmsetup', 'create', self.dm_node_name], table)
# Update the device_path but remember the old one for when we unlink the volume again
self.unlinked_device_path = self.device_path
self.device_path = self.dm_node_path
def _before_mount(self, e):
"""Mount the partition
"""
log_check_call([
'mount', '--types', self.filesystem, self.device_path,
e.destination
])
self.mount_dir = e.destination
def _before_create(self, e):
# Create the partition and then set the name of the partition afterwards
super(GPTPartition, self)._before_create(e)
# partition name only works for gpt, for msdos that becomes the part-type (primary, extended, logical)
name_command = ('name {idx} {name}'.format(idx=self.get_index(),
name=self.name))
log_check_call(
['parted', '--script', e.volume.device_path, '--', name_command])
def run(cls, info):
from base.fs.partitions.unformatted import UnformattedPartition
import re
# Disable the time based filesystem check
for partition in info.volume.partition_map.partitions:
if not isinstance(partition, UnformattedPartition):
if re.match('^ext[2-4]$', partition.filesystem) is not None:
log_check_call(['tune2fs', '-i', '0', partition.device_path])
def _before_create(self, e):
super(GPTPartition, self)._before_create(e)
# partition name only works for gpt, for msdos that becomes the part-type (primary, extended, logical)
name_command = ('name {idx} {name}'
.format(idx=self.get_index(),
name=self.name))
log_check_call(['/sbin/parted', '--script', e.volume.device_path,
'--', name_command])
def run(cls, info): log_check_call(['chroot', info.root, 'apt-get', 'clean']) lists = os.path.join(info.root, 'var/lib/apt/lists') for list_file in [os.path.join(lists, f) for f in os.listdir(lists)]: if os.path.isfile(list_file): os.remove(list_file)
def _before_unmap(self, event):
volume = event.volume
for partition in self.partitions:
if partition.fsm.cannot('unmap'):
msg = 'The partition {partition} prevents the unmap procedure'.format(partition=partition)
raise PartitionError(msg)
log_check_call(['/sbin/kpartx', '-d', volume.device_path])
for partition in self.partitions:
partition.unmap()
def unmount(self): """Performs the unmount operation or asks the partition to unmount itself """ # If its a partition, it can unmount itself if isinstance(self.source, AbstractPartition): self.source.unmount() else: log_check_call(['umount', self.mount_dir]) del self.mount_dir
def unmount(self):
"""Performs the unmount operation or asks the partition to unmount itself
"""
# If its a partition, it can unmount itself
if isinstance(self.source, AbstractPartition):
self.source.unmount()
else:
log_check_call(['umount', self.mount_dir])
del self.mount_dir
def _before_unmount(self, e): """Unmount any mounts associated with this partition """ # Unmount the mounts in descending order of mounpoint path length # You cannot unmount /dev before you have unmounted /dev/pts for destination in sorted(self.mounts.iterkeys(), key=len, reverse=True): self.mounts[destination].unmount() log_check_call(['umount', self.mount_dir]) del self.mount_dir
def _before_create(self, event): volume = event.volume # Disk alignment still plays a role in virtualized environment, # but I honestly have no clue as to what best practice is here, so we choose 'none' log_check_call(['parted', '--script', '--align', 'none', volume.device_path, '--', 'mklabel', 'msdos']) # Create the partitions for partition in self.partitions: partition.create(volume)
def run(cls, info):
bundle_name = 'bundle-{id}'.format(id=info.run_id)
info.bundle_path = os.path.join(info.workspace, bundle_name)
log_check_call(['/usr/bin/euca-bundle-image',
'--image', info.volume.image_path,
'--user', info.credentials['user-id'],
'--privatekey', info.credentials['private-key'],
'--cert', info.credentials['certificate'],
'--ec2cert', cert_ec2,
'--destination', info.bundle_path,
'--prefix', info.ami_name])
def _before_unmount(self, e):
"""Unmount any mounts associated with this partition
"""
# Unmount the mounts in descending order of mounpoint path length
# You cannot unmount /dev before you have unmounted /dev/pts
for destination in sorted(self.mounts.iterkeys(),
key=len,
reverse=True):
self.mounts[destination].unmount()
log_check_call(['umount', self.mount_dir])
del self.mount_dir
def run(cls, info):
ssh_hostkeys = ['ssh_host_dsa_key',
'ssh_host_rsa_key']
if info.manifest.system['release'] != 'squeeze':
ssh_hostkeys.append('ssh_host_ecdsa_key')
private = [os.path.join(info.root, 'etc/ssh', name) for name in ssh_hostkeys]
public = [path + '.pub' for path in private]
from common.tools import log_check_call
log_check_call(['shred', '--remove'] + private + public)
def run(cls, info):
ssh_hostkeys = ['ssh_host_dsa_key',
'ssh_host_rsa_key']
if info.manifest.system['release'] != 'squeeze':
ssh_hostkeys.append('ssh_host_ecdsa_key')
private = [os.path.join(info.root, 'etc/ssh', name) for name in ssh_hostkeys]
public = [path + '.pub' for path in private]
from common.tools import log_check_call
log_check_call(['/usr/bin/shred', '--remove'] + private + public)
def mount(self, prefix): """Performs the mount operation or forwards it to another partition Args: prefix (str): Path prefix of the mountpoint """ mount_dir = os.path.join(prefix, self.destination) # If the source is another partition, we tell that partition to mount itself if isinstance(self.source, AbstractPartition): self.source.mount(destination=mount_dir) else: log_check_call(['mount'] + self.opts + [self.source, mount_dir]) self.mount_dir = mount_dir
def _before_create(self, e):
from common.tools import log_check_call
create_command = ('mkpart primary {start} {end}'
.format(start=str(self.get_start()),
end=str(self.get_end())))
log_check_call(['/sbin/parted', '--script', '--align', 'none', e.volume.device_path,
'--', create_command])
for flag in self.flags:
log_check_call(['/sbin/parted', '--script', e.volume.device_path,
'--', ('set {idx} {flag} on'
.format(idx=str(self.get_index()), flag=flag))])
def _before_unlink_dm_node(self, e):
"""Unlinks the device mapping
Args:
e (_e_obj): Event object containing arguments to create()
"""
log_check_call(['dmsetup', 'remove', self.dm_node_name])
# Delete the no longer valid information
del self.dm_node_name
del self.dm_node_path
# Reset the device_path
self.device_path = self.unlinked_device_path
def _before_unlink_dm_node(self, e): """Unlinks the device mapping Args: e (_e_obj): Event object containing arguments to create() """ log_check_call(['dmsetup', 'remove', self.dm_node_name]) # Delete the no longer valid information del self.dm_node_name del self.dm_node_path # Reset the device_path self.device_path = self.unlinked_device_path
def run(cls, info):
import stat
rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP
| stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
from shutil import copy
for name, src in info.initd['install'].iteritems():
dst = os.path.join(info.root, 'etc/init.d', name)
copy(src, dst)
os.chmod(dst, rwxr_xr_x)
log_check_call(['chroot', info.root, 'insserv', '--default', name])
for name in info.initd['disable']:
log_check_call(['chroot', info.root, 'insserv', '--remove', name])
def mount(self, prefix):
"""Performs the mount operation or forwards it to another partition
Args:
prefix (str): Path prefix of the mountpoint
"""
mount_dir = os.path.join(prefix, self.destination)
# If the source is another partition, we tell that partition to mount itself
if isinstance(self.source, AbstractPartition):
self.source.mount(destination=mount_dir)
else:
log_check_call(['mount'] + self.opts +
[self.source, mount_dir])
self.mount_dir = mount_dir
def run(cls, info):
bundle_name = 'bundle-{id}'.format(id=info.run_id)
info.bundle_path = os.path.join(info.workspace, bundle_name)
arch = {'i386': 'i386', 'amd64': 'x86_64'}.get(info.manifest.system['architecture'])
log_check_call(['euca-bundle-image',
'--image', info.volume.image_path,
'--arch', arch,
'--user', info.credentials['user-id'],
'--privatekey', info.credentials['private-key'],
'--cert', info.credentials['certificate'],
'--ec2cert', cert_ec2,
'--destination', info.bundle_path,
'--prefix', info.ami_name])
def run(cls, info):
from subprocess import CalledProcessError
from common.tools import log_check_call
try:
log_check_call(['chroot', info.root,
'dpkg-query', '-W', 'openssh-server'])
from common.tools import sed_i
sshdconfig_path = os.path.join(info.root, 'etc/ssh/sshd_config')
sed_i(sshdconfig_path, 'PermitRootLogin yes', 'PermitRootLogin no')
except CalledProcessError:
import logging
logging.getLogger(__name__).warn('The OpenSSH server has not been installed, '
'not disabling SSH root login.')
def run(cls, info):
from common.fs.loopbackvolume import LoopbackVolume
from common.tools import log_check_call
boot_dir = os.path.join(info.root, 'boot')
grub_dir = os.path.join(boot_dir, 'grub')
from common.fs import remount
p_map = info.volume.partition_map
def link_fn():
info.volume.link_dm_node()
if isinstance(p_map, partitionmaps.none.NoPartitions):
p_map.root.device_path = info.volume.device_path
def unlink_fn():
info.volume.unlink_dm_node()
if isinstance(p_map, partitionmaps.none.NoPartitions):
p_map.root.device_path = info.volume.device_path
# GRUB cannot deal with installing to loopback devices
# so we fake a real harddisk with dmsetup.
# Guide here: http://ebroder.net/2009/08/04/installing-grub-onto-a-disk-image/
if isinstance(info.volume, LoopbackVolume):
remount(info.volume, link_fn)
try:
[device_path] = log_check_call(['readlink', '-f', info.volume.device_path])
device_map_path = os.path.join(grub_dir, 'device.map')
partition_prefix = 'msdos'
if isinstance(p_map, partitionmaps.gpt.GPTPartitionMap):
partition_prefix = 'gpt'
with open(device_map_path, 'w') as device_map:
device_map.write('(hd0) {device_path}\n'.format(device_path=device_path))
if not isinstance(p_map, partitionmaps.none.NoPartitions):
for idx, partition in enumerate(info.volume.partition_map.partitions):
device_map.write('(hd0,{prefix}{idx}) {device_path}\n'
.format(device_path=partition.device_path,
prefix=partition_prefix,
idx=idx + 1))
# Install grub
log_check_call(['/usr/sbin/chroot', info.root,
'/usr/sbin/grub-install', device_path])
log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub'])
except Exception as e:
if isinstance(info.volume, LoopbackVolume):
remount(info.volume, unlink_fn)
raise e
if isinstance(info.volume, LoopbackVolume):
remount(info.volume, unlink_fn)
def run(cls, info):
from common.fs.loopbackvolume import LoopbackVolume
from common.tools import log_check_call
boot_dir = os.path.join(info.root, 'boot')
grub_dir = os.path.join(boot_dir, 'grub')
from common.fs import remount
p_map = info.volume.partition_map
def link_fn():
info.volume.link_dm_node()
if isinstance(p_map, partitionmaps.none.NoPartitions):
p_map.root.device_path = info.volume.device_path
def unlink_fn():
info.volume.unlink_dm_node()
if isinstance(p_map, partitionmaps.none.NoPartitions):
p_map.root.device_path = info.volume.device_path
# GRUB cannot deal with installing to loopback devices
# so we fake a real harddisk with dmsetup.
# Guide here: http://ebroder.net/2009/08/04/installing-grub-onto-a-disk-image/
if isinstance(info.volume, LoopbackVolume):
remount(info.volume, link_fn)
try:
[device_path] = log_check_call(['readlink', '-f', info.volume.device_path])
device_map_path = os.path.join(grub_dir, 'device.map')
partition_prefix = 'msdos'
if isinstance(p_map, partitionmaps.gpt.GPTPartitionMap):
partition_prefix = 'gpt'
with open(device_map_path, 'w') as device_map:
device_map.write('(hd0) {device_path}\n'.format(device_path=device_path))
if not isinstance(p_map, partitionmaps.none.NoPartitions):
for idx, partition in enumerate(info.volume.partition_map.partitions):
device_map.write('(hd0,{prefix}{idx}) {device_path}\n'
.format(device_path=partition.device_path,
prefix=partition_prefix,
idx=idx + 1))
# Install grub
log_check_call(['chroot', info.root,
'grub-install', device_path])
log_check_call(['chroot', info.root, 'update-grub'])
except Exception as e:
if isinstance(info.volume, LoopbackVolume):
remount(info.volume, unlink_fn)
raise e
if isinstance(info.volume, LoopbackVolume):
remount(info.volume, unlink_fn)
def run(cls, info): import stat rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) from shutil import copy for name, src in info.initd['install'].iteritems(): dst = os.path.join(info.root, 'etc/init.d', name) copy(src, dst) os.chmod(dst, rwxr_xr_x) log_check_call(['chroot', info.root, 'insserv', '--default', name]) for name in info.initd['disable']: log_check_call(['chroot', info.root, 'insserv', '--remove', name])
def run(cls, info):
bundle_name = 'bundle-{id}'.format(id=info.run_id)
info.bundle_path = os.path.join(info.workspace, bundle_name)
arch = {
'i386': 'i386',
'amd64': 'x86_64'
}.get(info.manifest.system['architecture'])
log_check_call([
'euca-bundle-image', '--image', info.volume.image_path, '--arch',
arch, '--user', info.credentials['user-id'], '--privatekey',
info.credentials['private-key'], '--cert',
info.credentials['certificate'], '--ec2cert', cert_ec2,
'--destination', info.bundle_path, '--prefix', info.ami_name
])
def run(cls, info):
if 'metadata_sources' in info.manifest.plugins['cloud_init']:
sources = info.manifest.plugins['cloud_init']['metadata_sources']
else:
source_mapping = {'ec2': 'Ec2'}
sources = source_mapping.get(info.manifest.provider, None)
if sources is None:
import logging
msg = ('No cloud-init metadata source mapping found for provider `{provider}\', '
'skipping selections setting.').format(info.manifest.provider)
logging.getLogger(__name__).warn(msg)
return
sources = "cloud-init cloud-init/datasources multiselect " + sources
log_check_call(['chroot', info.root, 'debconf-set-selections'], sources)
def run(cls, info):
manifest_file = os.path.join(info.bundle_path, info.ami_name + '.manifest.xml')
if info.host['region'] == 'us-east-1':
s3_url = 'https://s3.amazonaws.com/'
else:
s3_url = 'https://s3-{region}.amazonaws.com/'.format(region=info.host['region'])
log_check_call(['/usr/bin/euca-upload-bundle',
'--bucket', info.manifest.image['bucket'],
'--manifest', manifest_file,
'--access-key', info.credentials['access-key'],
'--secret-key', info.credentials['secret-key'],
'--url', s3_url,
'--region', info.host['region'],
'--ec2cert', cert_ec2])
def run(cls, info):
init_scripts_dir = os.path.join(assets, 'init.d')
install = info.initd['install']
from subprocess import CalledProcessError
try:
log_check_call(['chroot', info.root,
'dpkg-query', '-W', 'openssh-server'])
if info.manifest.system['release'] == 'squeeze':
install['generate-ssh-hostkeys'] = os.path.join(init_scripts_dir, 'squeeze/generate-ssh-hostkeys')
else:
install['generate-ssh-hostkeys'] = os.path.join(init_scripts_dir, 'generate-ssh-hostkeys')
except CalledProcessError:
import logging
logging.getLogger(__name__).warn('The OpenSSH server has not been installed, '
'not installing SSH host key generation script.')
def run(cls, info):
from common.tools import sed_i
from common.tools import log_check_call
locale_gen = os.path.join(info.root, 'etc/locale.gen')
locale_str = '{locale}.{charmap} {charmap}'.format(locale=info.manifest.system['locale'],
charmap=info.manifest.system['charmap'])
search = '# ' + locale_str
sed_i(locale_gen, search, locale_str)
log_check_call(['chroot', info.root, 'locale-gen'])
lang = '{locale}.{charmap}'.format(locale=info.manifest.system['locale'],
charmap=info.manifest.system['charmap'])
log_check_call(['chroot', info.root,
'update-locale', 'LANG=' + lang])
def _before_format(self, e):
"""Formats the partition
"""
# If there is no explicit format_command define we simply call mkfs.fstype
if self.format_command is None:
format_command = ['mkfs.{fs}', '{device_path}']
else:
format_command = self.format_command
variables = {'fs': self.filesystem,
'device_path': self.device_path,
'size': self.size,
}
command = map(lambda part: part.format(**variables), format_command)
# Format the partition
log_check_call(command)
def run(cls, info):
from common.tools import sed_i
from common.tools import log_check_call
locale_gen = os.path.join(info.root, 'etc/locale.gen')
locale_str = '{locale}.{charmap} {charmap}'.format(
locale=info.manifest.system['locale'],
charmap=info.manifest.system['charmap'])
search = '# ' + locale_str
sed_i(locale_gen, search, locale_str)
log_check_call(['chroot', info.root, 'locale-gen'])
lang = '{locale}.{charmap}'.format(
locale=info.manifest.system['locale'],
charmap=info.manifest.system['charmap'])
log_check_call(['chroot', info.root, 'update-locale', 'LANG=' + lang])
def run(cls, info):
from common.tools import log_check_call
from subprocess import CalledProcessError
missing_packages = []
for package in info.host_dependencies:
try:
log_check_call(['/usr/bin/dpkg-query', '-s', package])
except CalledProcessError:
missing_packages.append(package)
if len(missing_packages) > 0:
pkgs = '\', `'.join(missing_packages)
if len(missing_packages) > 1:
msg = "The packages `{packages}\' are not installed".format(packages=pkgs)
else:
msg = "The package `{packages}\' is not installed".format(packages=pkgs)
raise TaskError(msg)
def _before_format(self, e):
"""Formats the partition
"""
# If there is no explicit format_command define we simply call mkfs.fstype
if self.format_command is None:
format_command = ['mkfs.{fs}', '{device_path}']
else:
format_command = self.format_command
variables = {
'fs': self.filesystem,
'device_path': self.device_path,
'size': self.size,
}
command = map(lambda part: part.format(**variables), format_command)
# Format the partition
log_check_call(command)
def _before_create(self, e):
"""Creates the partition
"""
from common.tools import log_check_call
# The create command is failry simple, start and end are just Bytes objects coerced into strings
create_command = ('mkpart primary {start} {end}'
.format(start=str(self.get_start()),
end=str(self.get_end())))
# Create the partition
log_check_call(['parted', '--script', '--align', 'none', e.volume.device_path,
'--', create_command])
# Set any flags on the partition
for flag in self.flags:
log_check_call(['parted', '--script', e.volume.device_path,
'--', ('set {idx} {flag} on'
.format(idx=str(self.get_index()), flag=flag))])
def get_uuid(self):
"""Gets the UUID of the partition
Returns:
str. The UUID of the partition
"""
[uuid] = log_check_call(
['blkid', '-s', 'UUID', '-o', 'value', self.device_path])
return uuid
def run(cls, info):
init_scripts_dir = os.path.join(assets, 'init.d')
install = info.initd['install']
from subprocess import CalledProcessError
try:
log_check_call(
['chroot', info.root, 'dpkg-query', '-W', 'openssh-server'])
if info.manifest.system['release'] == 'squeeze':
install['generate-ssh-hostkeys'] = os.path.join(
init_scripts_dir, 'squeeze/generate-ssh-hostkeys')
else:
install['generate-ssh-hostkeys'] = os.path.join(
init_scripts_dir, 'generate-ssh-hostkeys')
except CalledProcessError:
import logging
logging.getLogger(__name__).warn(
'The OpenSSH server has not been installed, '
'not installing SSH host key generation script.')
def run(cls, info):
info.packages.add('bzip2')
info.packages.add('build-essential')
info.packages.add('dkms')
from common.tools import log_check_call
[kernel_version] = log_check_call(['chroot', info.root, 'uname', '-r'])
kernel_headers_pkg = 'linux-headers-{version}'.format(
version=kernel_version)
info.packages.add(kernel_headers_pkg)
def run(cls, info):
manifest_file = os.path.join(info.bundle_path,
info.ami_name + '.manifest.xml')
if info.host['region'] == 'us-east-1':
s3_url = 'https://s3.amazonaws.com/'
elif info.host['region'] == 'cn-north-1':
s3_url = 'https://s3.cn-north-1.amazonaws.com.cn'
else:
s3_url = 'https://s3-{region}.amazonaws.com/'.format(
region=info.host['region'])
info.manifest.manifest_location = info.manifest.image[
'bucket'] + '/' + info.ami_name + '.manifest.xml'
log_check_call([
'euca-upload-bundle', '--bucket', info.manifest.image['bucket'],
'--manifest', manifest_file, '--access-key',
info.credentials['access-key'], '--secret-key',
info.credentials['secret-key'], '--url', s3_url, '--region',
info.host['region'], '--ec2cert', cert_ec2
])
def run(cls, info):
import stat
rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP
| stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
x_all = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
grubd = os.path.join(info.root, 'etc/grub.d')
for cfg in [os.path.join(grubd, f) for f in os.listdir(grubd)]:
os.chmod(cfg, os.stat(cfg).st_mode & ~x_all)
from shutil import copy
script_src = os.path.join(assets, 'grub.d/40_custom')
script_dst = os.path.join(info.root, 'etc/grub.d/40_custom')
copy(script_src, script_dst)
os.chmod(script_dst, rwxr_xr_x)
from base.fs.partitionmaps.none import NoPartitions
if not isinstance(info.volume.partition_map, NoPartitions):
from common.tools import sed_i
root_idx = info.volume.partition_map.root.get_index()
grub_device = 'GRUB_DEVICE=/dev/xvda{idx}'.format(idx=root_idx)
sed_i(script_dst, '^GRUB_DEVICE=/dev/xvda$', grub_device)
grub_root = '\troot (hd0,{idx})'.format(idx=root_idx - 1)
sed_i(script_dst, '^\troot \(hd0\)$', grub_root)
if info.manifest.volume['backing'] == 's3':
from common.tools import sed_i
sed_i(script_dst, '^GRUB_DEVICE=/dev/xvda$',
'GRUB_DEVICE=/dev/xvda1')
from common.tools import sed_i
grub_def = os.path.join(info.root, 'etc/default/grub')
sed_i(grub_def, '^GRUB_TIMEOUT=[0-9]+', 'GRUB_TIMEOUT=0\n'
'GRUB_HIDDEN_TIMEOUT=true')
from common.tools import log_check_call
log_check_call(['chroot', info.root, 'update-grub'])
log_check_call([
'chroot', info.root, 'ln', '--symbolic', '/boot/grub/grub.cfg',
'/boot/grub/menu.lst'
])
def run(cls, info):
with open(os.path.join(info.root, 'etc/hostname')) as handle:
hostname = handle.read().strip()
with open(os.path.join(info.root, 'etc/hosts'), 'a') as handle:
handle.write('127.0.0.1\t{hostname}\n'.format(hostname=hostname))
from shutil import copy
pp_manifest = info.manifest.plugins['puppet']['manifest']
manifest_rel_dst = os.path.join('tmp', os.path.basename(pp_manifest))
manifest_dst = os.path.join(info.root, manifest_rel_dst)
copy(pp_manifest, manifest_dst)
manifest_path = os.path.join('/', manifest_rel_dst)
from common.tools import log_check_call
log_check_call(['chroot', info.root,
'puppet', 'apply', manifest_path])
os.remove(manifest_dst)
from common.tools import sed_i
hosts_path = os.path.join(info.root, 'etc/hosts')
sed_i(hosts_path, '127.0.0.1\s*{hostname}\n?'.format(hostname=hostname), '')
def run(cls, info):
from common.tools import log_check_call
from subprocess import CalledProcessError
import re
missing_packages = []
for command, package in info.host_dependencies.items():
try:
log_check_call(['type ' + command], shell=True)
except CalledProcessError:
if re.match('^https?:\/\/', package):
msg = ('The command `{command}\' is not available, '
'you can download the software at `{package}\'.'.
format(command=command, package=package))
else:
msg = ('The command `{command}\' is not available, '
'it is located in the package `{package}\'.'.format(
command=command, package=package))
missing_packages.append(msg)
if len(missing_packages) > 0:
msg = '\n'.join(missing_packages)
raise TaskError(msg)
def run(cls, info):
ssh_dir = os.path.join(info.root, 'home/vagrant/.ssh')
os.mkdir(ssh_dir)
authorized_keys_source_path = os.path.join(assets, 'authorized_keys')
with open(authorized_keys_source_path, 'r') as authorized_keys_source:
insecure_public_key = authorized_keys_source.read()
authorized_keys_path = os.path.join(ssh_dir, 'authorized_keys')
with open(authorized_keys_path, 'a') as authorized_keys:
authorized_keys.write(insecure_public_key)
import stat
os.chmod(ssh_dir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
os.chmod(authorized_keys_path, stat.S_IRUSR | stat.S_IWUSR)
# We can't do this directly with python, since getpwnam gets its info from the host
from common.tools import log_check_call
log_check_call([
'chroot', info.root, 'chown', 'vagrant:vagrant',
'/home/vagrant/.ssh', '/home/vagrant/.ssh/authorized_keys'
])
def install_local(cls, info, local_packages):
from shutil import copy
from common.tools import log_check_call
import os
absolute_package_paths = []
chrooted_package_paths = []
for package_src in local_packages:
pkg_name = os.path.basename(package_src.path)
package_rel_dst = os.path.join('tmp', pkg_name)
package_dst = os.path.join(info.root, package_rel_dst)
copy(package_src.path, package_dst)
absolute_package_paths.append(package_dst)
package_path = os.path.join('/', package_rel_dst)
chrooted_package_paths.append(package_path)
env = os.environ.copy()
env['DEBIAN_FRONTEND'] = 'noninteractive'
log_check_call(['chroot', info.root, 'dpkg', '--install'] +
chrooted_package_paths,
env=env)
for path in absolute_package_paths:
os.remove(path)
