def mount_target(self, chroot, treat_mtab=True, pseudo=True):
"""Mount a set of file systems into a chroot
:param chroot: Directory where to mount file systems
:param treat_mtab: If mtab needs to be actualized (Default: True)
:param pseudo: If pseudo file systems
need to be mounted (Default: True)
"""
LOG.debug('Mounting target file systems: %s', chroot)
# Here we are going to mount all file systems in partition scheme.
for fs in self.driver.partition_scheme.fs_sorted_by_depth():
if fs.mount == 'swap':
continue
mount = chroot + fs.mount
utils.makedirs_if_not_exists(mount)
fu.mount_fs(fs.type, str(fs.device), mount)
if pseudo:
for path in ('/sys', '/dev', '/proc'):
utils.makedirs_if_not_exists(chroot + path)
fu.mount_bind(chroot, path)
if treat_mtab:
mtab = utils.execute(
'chroot', chroot, 'grep', '-v', 'rootfs', '/proc/mounts')[0]
mtab_path = chroot + '/etc/mtab'
if os.path.islink(mtab_path):
os.remove(mtab_path)
with open(mtab_path, 'wt', encoding='utf-8') as f:
f.write(six.text_type(mtab))
def _mount_target(self,
mount_dir,
os_id=None,
pseudo=True,
treat_mtab=True):
LOG.debug('Mounting target file systems: %s', mount_dir)
# Here we are going to mount all file systems in partition schema.
for fs in self.driver.partition_scheme.fs_sorted_by_depth(os_id):
if fs.mount == 'swap':
continue
mount = os.path.join(mount_dir, fs.mount.strip(os.sep))
utils.makedirs_if_not_exists(mount)
fu.mount_fs(fs.type, str(fs.device), mount)
if pseudo:
for path in ('/sys', '/dev', '/proc'):
utils.makedirs_if_not_exists(
os.path.join(mount_dir, path.strip(os.sep)))
fu.mount_bind(mount_dir, path)
if treat_mtab:
mtab = utils.execute('chroot', mount_dir, 'grep', '-v', 'rootfs',
'/proc/mounts')[0]
mtab_path = os.path.join(mount_dir, 'etc/mtab')
if os.path.islink(mtab_path):
os.remove(mtab_path)
with open(mtab_path, 'wt', encoding='utf-8') as f:
f.write(six.text_type(mtab))
def copy_kernel_initramfs(chroot, dstdir, clean=False):
"""Copy latest or newest vmlinuz and initrd from chroot
:param chroot:
:param dstdir: copy to folder
:param clean: remove all vmlinuz\initrd after done
:return:
"""
# TODO(azvyagintsev) fetch from uri driver
# module* : result filename
files = {'vmlinuz': 'vmlinuz',
'initrd': 'initrd.img'
}
utils.makedirs_if_not_exists(dstdir)
boot_dir = os.path.join(chroot, 'boot')
for module in six.iterkeys(files):
mask = os.path.join(boot_dir, module + '*')
all_files = glob.glob(mask)
if len(all_files) > 1:
raise errors.TooManyKernels(
"Too many %s detected :%s", module, all_files)
file_to_copy = all_files[0]
copy_to = os.path.join(dstdir, files[module])
LOG.debug('Copying file: %s to: %s', file_to_copy, copy_to)
shutil.copy(file_to_copy, copy_to)
if clean:
files_to_remove = glob.glob(mask)
remove_files('/', files_to_remove)
def _mount_target(self, mount_dir, os_id=None, pseudo=True,
treat_mtab=True):
LOG.debug('Mounting target file systems: %s', mount_dir)
# Here we are going to mount all file systems in partition schema.
for fs in self.driver.partition_scheme.fs_sorted_by_depth(os_id):
if fs.mount == 'swap':
continue
mount = os.path.join(mount_dir, fs.mount.strip(os.sep))
utils.makedirs_if_not_exists(mount)
fu.mount_fs(fs.type, str(fs.device), mount)
if pseudo:
for path in ('/sys', '/dev', '/proc'):
utils.makedirs_if_not_exists(
os.path.join(mount_dir, path.strip(os.sep)))
fu.mount_bind(mount_dir, path)
if treat_mtab:
mtab = utils.execute('chroot', mount_dir, 'grep', '-v', 'rootfs',
'/proc/mounts')[0]
mtab_path = os.path.join(mount_dir, 'etc/mtab')
if os.path.islink(mtab_path):
os.remove(mtab_path)
with open(mtab_path, 'wt', encoding='utf-8') as f:
f.write(six.text_type(mtab))
def mount_target(self, chroot, treat_mtab=True, pseudo=True):
"""Mount a set of file systems into a chroot
:param chroot: Directory where to mount file systems
:param treat_mtab: If mtab needs to be actualized (Default: True)
:param pseudo: If pseudo file systems
need to be mounted (Default: True)
"""
LOG.debug('Mounting target file systems: %s', chroot)
# Here we are going to mount all file systems in partition scheme.
for fs in self.driver.partition_scheme.fs_sorted_by_depth():
if fs.mount == 'swap':
continue
mount = chroot + fs.mount
utils.makedirs_if_not_exists(mount)
fu.mount_fs(fs.type, str(fs.device), mount)
if pseudo:
for path in ('/sys', '/dev', '/proc'):
utils.makedirs_if_not_exists(chroot + path)
fu.mount_bind(chroot, path)
if treat_mtab:
mtab = utils.execute('chroot', chroot, 'grep', '-v', 'rootfs',
'/proc/mounts')[0]
mtab_path = chroot + '/etc/mtab'
if os.path.islink(mtab_path):
os.remove(mtab_path)
with open(mtab_path, 'wt', encoding='utf-8') as f:
f.write(six.text_type(mtab))
def save_bs_container(output, input_dir, format="tar.gz"):
"""Copy files from dir to archive or another directory
:param output:
:param input_dir:
:param format:
:return:
"""
if format == 'directory':
utils.makedirs_if_not_exists(output)
bs_files = os.listdir(input_dir)
LOG.debug("Output folder: %s\ntry to copy bootstrap files: %s", output,
bs_files)
for bs_file in bs_files:
abs_bs_file = os.path.join(input_dir, bs_file)
if (os.path.isfile(abs_bs_file)):
if os.path.isfile(os.path.join(output, bs_file)):
raise errors.BootstrapFileAlreadyExists(
"File: {0} already exists in: {1}".format(
bs_file, output))
shutil.copy(abs_bs_file, output)
os.chmod(os.path.join(output, bs_file), 0o755)
return output
elif format == 'tar.gz':
LOG.debug("Try to make output archive file: %s", output)
output = make_targz(input_dir, output_name=output)
return output
else:
raise errors.WrongOutputContainer(
"Unsupported bootstrap container format {0}.".format(format))
def copy_kernel_initramfs(chroot, dstdir, clean=False):
"""Copy latest or newest vmlinuz and initrd from chroot
:param chroot:
:param dstdir: copy to folder
:param clean: remove all vmlinuz\initrd after done
:return:
"""
# TODO(azvyagintsev) fetch from uri driver
# module* : result filename
files = {'vmlinuz': 'vmlinuz', 'initrd': 'initrd.img'}
utils.makedirs_if_not_exists(dstdir)
boot_dir = os.path.join(chroot, 'boot')
for module in six.iterkeys(files):
mask = os.path.join(boot_dir, module + '*')
all_files = glob.glob(mask)
if len(all_files) > 1:
raise errors.TooManyKernels("Too many %s detected :%s", module,
all_files)
file_to_copy = all_files[0]
copy_to = os.path.join(dstdir, files[module])
LOG.debug('Copying file: %s to: %s', file_to_copy, copy_to)
shutil.copy(file_to_copy, copy_to)
if clean:
files_to_remove = glob.glob(mask)
remove_files('/', files_to_remove)
def save_bs_container(output, input_dir, format="tar.gz"):
"""Copy files from dir to archive or another directory
:param output:
:param input_dir:
:param format:
:return:
"""
if format == 'directory':
utils.makedirs_if_not_exists(output)
bs_files = os.listdir(input_dir)
LOG.debug("Output folder: %s\ntry to copy bootstrap files: %s",
output, bs_files)
for bs_file in bs_files:
abs_bs_file = os.path.join(input_dir, bs_file)
if (os.path.isfile(abs_bs_file)):
if os.path.isfile(os.path.join(output, bs_file)):
raise errors.BootstrapFileAlreadyExists(
"File: {0} already exists in: {1}"
.format(bs_file, output))
shutil.copy(abs_bs_file, output)
os.chmod(os.path.join(output, bs_file), 0o755)
return output
elif format == 'tar.gz':
LOG.debug("Try to make output archive file: %s", output)
output = make_targz(input_dir, output_name=output)
return output
else:
raise errors.WrongOutputContainer(
"Unsupported bootstrap container format {0}."
.format(format))
def mkdtemp_smart(root_dir, suffix):
"""Create a unique temporary directory in root_dir
Automatically creates root_dir if it does not exist.
Otherwise same as tempfile.mkdtemp
"""
LOG.debug('Creating temporary chroot directory')
utils.makedirs_if_not_exists(root_dir)
chroot = tempfile.mkdtemp(dir=root_dir, suffix=suffix)
LOG.debug('Temporary chroot dir: %s', chroot)
return chroot
def restore_resolv_conf(chroot):
"""Restore hosts/resolv files in chroot
opposite to propagate_host_resolv_conf
"""
c_etc = os.path.join(chroot, 'etc/')
utils.makedirs_if_not_exists(c_etc)
for conf_name in ('resolv.conf', 'hosts'):
dst_conf_name = os.path.join(c_etc, conf_name)
if os.path.isfile(dst_conf_name + '.bak'):
LOG.info('Restoring default {0} inside chroot'.format(conf_name))
shutil.move(dst_conf_name + '.bak', dst_conf_name)
def restore_resolv_conf(chroot):
"""Restore hosts/resolv files in chroot
opposite to propagate_host_resolv_conf
"""
c_etc = os.path.join(chroot, 'etc/')
utils.makedirs_if_not_exists(c_etc)
for conf_name in ('resolv.conf', 'hosts'):
dst_conf_name = os.path.join(c_etc, conf_name)
if os.path.isfile(dst_conf_name + '.bak'):
LOG.info('Restoring default {0} inside chroot'.
format(conf_name))
shutil.move(dst_conf_name + '.bak', dst_conf_name)
def _mount_bootloader(self, mount_dir):
fs = filter(lambda fss: fss.mount == 'multiboot',
self.driver.partition_scheme.fss)
if len(fs) > 1:
raise errors.WrongPartitionSchemeError(
'Multiple multiboot partitions found')
utils.makedirs_if_not_exists(mount_dir)
fu.mount_fs(fs[0].type, str(fs[0].device), mount_dir)
yield pu.get_uuid(fs[0].device)
fu.umount_fs(mount_dir)
def mkdtemp_smart(root_dir, suffix):
"""Create a unique temporary directory in root_dir
Automatically creates root_dir if it does not exist.
Otherwise same as tempfile.mkdtemp
"""
LOG.debug('Creating temporary chroot directory')
utils.makedirs_if_not_exists(root_dir)
chroot = tempfile.mkdtemp(
dir=root_dir, suffix=suffix)
LOG.debug('Temporary chroot dir: %s', chroot)
return chroot
def _mount_bootloader(self, mount_dir):
fs = filter(lambda fss: fss.mount == 'multiboot',
self.driver.partition_scheme.fss)
if len(fs) > 1:
raise errors.WrongPartitionSchemeError(
'Multiple multiboot partitions found')
utils.makedirs_if_not_exists(mount_dir)
fu.mount_fs(fs[0].type, str(fs[0].device), mount_dir)
yield pu.get_uuid(fs[0].device)
fu.umount_fs(mount_dir)
def make_targz(source_dir, output_name=None):
"""Archive the given directory
:param source_dir: directory to archive
:param output_name: output file name, might be a relative
or an absolute path
"""
if not output_name:
output_name = six.text_type(uuid.uuid4()) + '.tar.gz'
utils.makedirs_if_not_exists(os.path.dirname(output_name))
LOG.info('Creating archive: %s', output_name)
utils.execute('tar', '-czf', output_name, '--directory',
os.path.normcase(source_dir), '.', logged=True)
return output_name
def rsync_inject(src, dst):
"""Recursively copy the src to dst using full source paths
Example: suppose the source directory looks like
src/etc/myconfig
src/usr/bin/myscript
rsync_inject('src', '/tmp/chroot')
copies src/etc/myconfig to /tmp/chroot/etc/myconfig,
and src/usr/bin/myscript to /tmp/chroot/usr/bin/myscript,
respectively
"""
utils.makedirs_if_not_exists(os.path.dirname(dst))
LOG.debug('Rsync files from %s to: %s', src, dst)
utils.execute('rsync', '-rlptDKv', src + '/', dst + '/', logged=True)
def propagate_host_resolv_conf(chroot):
"""Copy DNS settings from host system to chroot.
Make a backup of original /etc/resolv.conf and /etc/hosts.
# In case user pass some custom rules in hosts\resolv.conf.
opposite to restore_resolv_conf
"""
c_etc = os.path.join(chroot, 'etc/')
utils.makedirs_if_not_exists(c_etc)
for conf_name in ('resolv.conf', 'hosts'):
dst_conf_name = os.path.join(c_etc, conf_name)
src_conf_name = os.path.join('/etc/', conf_name)
files_to_copy = [(dst_conf_name, dst_conf_name + '.bak'),
(src_conf_name, dst_conf_name)]
for src, dst in files_to_copy:
if os.path.isfile(src):
shutil.copy(src, dst)
def rsync_inject(src, dst):
"""Recursively copy the src to dst using full source paths
Example: suppose the source directory looks like
src/etc/myconfig
src/usr/bin/myscript
rsync_inject('src', '/tmp/chroot')
copies src/etc/myconfig to /tmp/chroot/etc/myconfig,
and src/usr/bin/myscript to /tmp/chroot/usr/bin/myscript,
respectively
"""
utils.makedirs_if_not_exists(os.path.dirname(dst))
LOG.debug('Rsync files from %s to: %s', src, dst)
utils.execute('rsync', '-rlptDKv', src + '/',
dst + '/', logged=True)
def dump_runtime_uuid(uuid, config):
"""Save runtime_uuid into yaml file
Simple uuid variable to identify bootstrap.
Variable will be hard-coded into config yaml file, in build-time
:param uuid:
:param config: yaml file
:return:
"""
data = {}
utils.makedirs_if_not_exists(os.path.dirname(config))
if os.path.isfile(config):
with open(config, 'r') as f:
data = yaml.load(f)
data['runtime_uuid'] = uuid
LOG.debug('Save runtime_uuid:%s to file: %s', uuid, config)
with open(config, 'wt') as f:
yaml.safe_dump(data, stream=f, encoding='utf-8')
def propagate_host_resolv_conf(chroot):
"""Copy DNS settings from host system to chroot.
Make a backup of original /etc/resolv.conf and /etc/hosts.
# In case user pass some custom rules in hosts\resolv.conf.
opposite to restore_resolv_conf
"""
c_etc = os.path.join(chroot, 'etc/')
utils.makedirs_if_not_exists(c_etc)
for conf_name in ('resolv.conf', 'hosts'):
dst_conf_name = os.path.join(c_etc, conf_name)
src_conf_name = os.path.join('/etc/', conf_name)
files_to_copy = [(dst_conf_name, dst_conf_name + '.bak'),
(src_conf_name, dst_conf_name)]
for src, dst in files_to_copy:
if os.path.isfile(src):
shutil.copy(src, dst)
def dump_runtime_uuid(uuid, config):
"""Save runtime_uuid into yaml file
Simple uuid variable to identify bootstrap.
Variable will be hard-coded into config yaml file, in build-time
:param uuid:
:param config: yaml file
:return:
"""
data = {}
utils.makedirs_if_not_exists(os.path.dirname(config))
if os.path.isfile(config):
with open(config, 'r') as f:
data = yaml.load(f)
data['runtime_uuid'] = uuid
LOG.debug('Save runtime_uuid:%s to file: %s', uuid, config)
with open(config, 'wt') as f:
yaml.safe_dump(data, stream=f, encoding='utf-8')
def make_targz(source_dir, output_name=None):
"""Archive the given directory
:param source_dir: directory to archive
:param output_name: output file name, might be a relative
or an absolute path
"""
if not output_name:
output_name = six.text_type(uuid.uuid4()) + '.tar.gz'
utils.makedirs_if_not_exists(os.path.dirname(output_name))
LOG.info('Creating archive: %s', output_name)
utils.execute('tar',
'-czf',
output_name,
'--directory',
os.path.normcase(source_dir),
'.',
logged=True)
return output_name
def run_mksquashfs(chroot, output_name=None, compression_algorithm='xz'):
"""Pack the target system as squashfs using mksquashfs
:param chroot: chroot system, to be squashfs'd
:param output_name: output file name, might be a relative
or an absolute path
The kernel squashfs driver has to match with the user space squasfs tools.
Use the mksquashfs provided by the target distro to achieve this.
(typically the distro maintainers are smart enough to ship the correct
version of mksquashfs)
Use mksquashfs installed in the target system
1)Mount tmpfs under chroot/mnt
2)run mksquashfs inside a chroot
3)move result files to dstdir
"""
if not output_name:
output_name = 'root.squashfs' + six.text_type(uuid.uuid4())
utils.makedirs_if_not_exists(os.path.dirname(output_name))
dstdir = os.path.dirname(output_name)
temp = '.mksquashfs.tmp.' + six.text_type(uuid.uuid4())
s_dst = os.path.join(chroot, 'mnt/dst')
s_src = os.path.join(chroot, 'mnt/src')
try:
fu.mount_fs('tmpfs', 'mnt_{0}'.format(temp),
(os.path.join(chroot, 'mnt')),
'rw,nodev,nosuid,noatime,mode=0755,size=4M')
utils.makedirs_if_not_exists(s_src)
utils.makedirs_if_not_exists(s_dst)
# Bind mount the chroot to avoid including various temporary/virtual
# files (/proc, /sys, /dev, and so on) into the image
fu.mount_fs(None, chroot, s_src, opts='bind')
fu.mount_fs(None, None, s_src, 'remount,bind,ro')
fu.mount_fs(None, dstdir, s_dst, opts='bind')
# run mksquashfs
chroot_squash = os.path.join('/mnt/dst/' + temp)
long_squash = os.path.join(chroot, 'mnt/dst/{0}'.format(temp))
utils.execute('chroot',
chroot,
'mksquashfs',
'/mnt/src',
chroot_squash,
'-comp',
compression_algorithm,
'-no-progress',
'-noappend',
logged=True)
# move to result name
LOG.debug('Moving file: %s to: %s', long_squash, output_name)
shutil.move(long_squash, output_name)
except Exception as exc:
LOG.error('squashfs_image build failed: %s', exc)
raise
finally:
LOG.info('squashfs_image clean-up')
stop_chrooted_processes(chroot, signal=signal.SIGTERM)
fu.umount_fs(os.path.join(chroot, 'mnt/dst'))
fu.umount_fs(os.path.join(chroot, 'mnt/src'))
fu.umount_fs(os.path.join(chroot, 'mnt'))
def run_mksquashfs(chroot, output_name=None, compression_algorithm='xz'):
"""Pack the target system as squashfs using mksquashfs
:param chroot: chroot system, to be squashfs'd
:param output_name: output file name, might be a relative
or an absolute path
The kernel squashfs driver has to match with the user space squasfs tools.
Use the mksquashfs provided by the target distro to achieve this.
(typically the distro maintainers are smart enough to ship the correct
version of mksquashfs)
Use mksquashfs installed in the target system
1)Mount tmpfs under chroot/mnt
2)run mksquashfs inside a chroot
3)move result files to dstdir
"""
if not output_name:
output_name = 'root.squashfs' + six.text_type(uuid.uuid4())
utils.makedirs_if_not_exists(os.path.dirname(output_name))
dstdir = os.path.dirname(output_name)
temp = '.mksquashfs.tmp.' + six.text_type(uuid.uuid4())
s_dst = os.path.join(chroot, 'mnt/dst')
s_src = os.path.join(chroot, 'mnt/src')
try:
fu.mount_fs(
'tmpfs', 'mnt_{0}'.format(temp),
(os.path.join(chroot, 'mnt')),
'rw,nodev,nosuid,noatime,mode=0755,size=4M')
utils.makedirs_if_not_exists(s_src)
utils.makedirs_if_not_exists(s_dst)
# Bind mount the chroot to avoid including various temporary/virtual
# files (/proc, /sys, /dev, and so on) into the image
fu.mount_fs(None, chroot, s_src, opts='bind')
fu.mount_fs(None, None, s_src, 'remount,bind,ro')
fu.mount_fs(None, dstdir, s_dst, opts='bind')
# run mksquashfs
chroot_squash = os.path.join('/mnt/dst/' + temp)
long_squash = os.path.join(chroot, 'mnt/dst/{0}'.format(temp))
utils.execute(
'chroot', chroot, 'mksquashfs', '/mnt/src',
chroot_squash,
'-comp', compression_algorithm,
'-no-progress', '-noappend', logged=True)
# move to result name
LOG.debug('Moving file: %s to: %s', long_squash, output_name)
shutil.move(long_squash, output_name)
except Exception as exc:
LOG.error('squashfs_image build failed: %s', exc)
raise
finally:
LOG.info('squashfs_image clean-up')
stop_chrooted_processes(chroot, signal=signal.SIGTERM)
fu.umount_fs(os.path.join(chroot, 'mnt/dst'))
fu.umount_fs(os.path.join(chroot, 'mnt/src'))
fu.umount_fs(os.path.join(chroot, 'mnt'))
def do_bootloader(self):
LOG.debug('--- Installing bootloader (do_bootloader) ---')
chroot = '/tmp/target'
partition_scheme = self.driver.partition_scheme
with self.mount_target(chroot):
mount2uuid = {}
for fs in partition_scheme.fss:
mount2uuid[fs.mount] = utils.execute(
'blkid', '-o', 'value', '-s', 'UUID', fs.device,
check_exit_code=[0])[0].strip()
if '/' not in mount2uuid:
raise errors.WrongPartitionSchemeError(
'Error: device with / mountpoint has not been found')
grub = self.driver.grub
guessed_version = gu.guess_grub_version(chroot=chroot)
if guessed_version != grub.version:
grub.version = guessed_version
LOG.warning('Grub version differs from which the operating '
'system should have by default. Found version in '
'image: %s', guessed_version)
boot_device = partition_scheme.boot_device(grub.version)
install_devices = [d.name for d in partition_scheme.parteds
if d.install_bootloader]
grub.append_kernel_params('root=UUID=%s ' % mount2uuid['/'])
kernel = grub.kernel_name or \
gu.guess_kernel(chroot=chroot, regexp=grub.kernel_regexp)
initrd = grub.initrd_name or \
gu.guess_initrd(chroot=chroot, regexp=grub.initrd_regexp)
if grub.version == 1:
gu.grub1_cfg(kernel=kernel, initrd=initrd,
kernel_params=grub.kernel_params, chroot=chroot,
grub_timeout=CONF.timeout)
gu.grub1_install(install_devices, boot_device, chroot=chroot)
else:
# TODO(kozhukalov): implement which kernel to use by default
# Currently only grub1_cfg accepts kernel and initrd
# parameters.
gu.grub2_cfg(kernel_params=grub.kernel_params, chroot=chroot,
grub_timeout=CONF.timeout)
gu.grub2_install(install_devices, chroot=chroot)
# TODO(agordeev): move to separate actions?
if CONF.fix_udev_net_rules:
# FIXME(agordeev) There's no convenient way to perfrom NIC
# remapping in Ubuntu, so injecting files prior the first boot
# should work
with open(chroot + '/etc/udev/rules.d/70-persistent-net.rules',
'wt', encoding='utf-8') as f:
f.write(u'# Generated by bareon during provisioning: '
u'BEGIN\n')
# pattern is aa:bb:cc:dd:ee:ff_eth0,aa:bb:cc:dd:ee:ff_eth1
for mapping in self.driver.configdrive_scheme. \
common.udevrules.split(','):
mac_addr, nic_name = mapping.split('_')
f.write(u'SUBSYSTEM=="net", ACTION=="add", '
u'DRIVERS=="?*", ATTR{address}=="%s", '
u'ATTR{type}=="1", KERNEL=="eth*", '
u'NAME="%s"\n' % (mac_addr, nic_name))
f.write(
u'# Generated by bareon during provisioning: END\n')
# FIXME(agordeev): Disable net-generator that adds new entries
# to 70-persistent-net.rules
with open(chroot + '/etc/udev/rules.d/'
'75-persistent-net-generator.rules', 'wt',
encoding='utf-8') as f:
f.write(u'# Generated by bareon during provisioning:\n'
u'# DO NOT DELETE. It is needed to disable '
u'net-generator\n')
# FIXME(kozhukalov): Prevent nailgun-agent from doing anything.
# This ugly hack is to be used together with the command removing
# this lock file not earlier than /etc/rc.local
# The reason for this hack to appear is to prevent nailgun-agent
# from changing mcollective config at the same time when cloud-init
# does the same. Otherwise, we can end up with corrupted
# mcollective config.
# For details see https://bugs.launchpad.net/fuel/+bug/1449186
LOG.debug('Preventing nailgun-agent from doing '
'anything until it is unlocked')
utils.makedirs_if_not_exists(os.path.join(chroot,
'etc/nailgun-agent'))
with open(os.path.join(chroot, 'etc/nailgun-agent/nodiscover'),
'w'):
pass
# FIXME(kozhukalov): When we have just os-root fs image and don't
# have os-var-log fs image while / and /var/log are supposed to be
# separate file systems and os-var-log is mounted into
# non-empty directory on the / file system, those files in /var/log
# directory become unavailable.
# The thing is that among those file there is /var/log/upstart
# where upstart daemon writes its logs. We have specific upstart
# job which is to flush open files once all file systems are
# mounted.
# This job needs upstart directory to be available on os-var-log
# file system.
# This is just a temporary fix and correct fix will be available
# soon via updates.
utils.execute('mkdir', '-p', chroot + '/var/log/upstart')
with open(chroot + '/etc/fstab', 'wt', encoding='utf-8') as f:
for fs in self.driver.partition_scheme.fss:
# TODO(kozhukalov): Think of improving the logic so as to
# insert a meaningful fsck order value which is last zero
# at fstab line. Currently we set it into 0 which means
# a corresponding file system will never be checked. We
# assume puppet or other configuration tool will care of
# it.
if fs.mount == '/':
f.write(u'UUID=%s %s %s defaults,errors=panic 0 0\n' %
(mount2uuid[fs.mount], fs.mount, fs.type))
else:
f.write(u'UUID=%s %s %s defaults 0 0\n' %
(mount2uuid[fs.mount], fs.mount, fs.type))
def do_bootloader(self):
LOG.debug('--- Installing bootloader (do_bootloader) ---')
chroot = '/tmp/target'
partition_scheme = self.driver.partition_scheme
with self.mount_target(chroot):
mount2uuid = {}
for fs in partition_scheme.fss:
mount2uuid[fs.mount] = utils.execute(
'blkid', '-c', '/dev/null', '-o', 'value', '-s', 'UUID',
fs.device, check_exit_code=[0])[0].strip()
if '/' not in mount2uuid:
raise errors.WrongPartitionSchemeError(
'Error: device with / mountpoint has not been found')
self._override_lvm_config(chroot)
grub = self.driver.grub
guessed_version = gu.guess_grub_version(chroot=chroot)
if guessed_version != grub.version:
grub.version = guessed_version
LOG.warning('Grub version differs from which the operating '
'system should have by default. Found version in '
'image: %s', guessed_version)
boot_device = partition_scheme.boot_device(grub.version)
install_devices = [d.name for d in partition_scheme.parteds
if d.install_bootloader]
grub.append_kernel_params('root=UUID=%s ' % mount2uuid['/'])
kernel = grub.kernel_name or \
gu.guess_kernel(chroot=chroot, regexp=grub.kernel_regexp)
initrd = grub.initrd_name or \
gu.guess_initrd(chroot=chroot, regexp=grub.initrd_regexp)
if grub.version == 1:
gu.grub1_cfg(kernel=kernel, initrd=initrd,
kernel_params=grub.kernel_params, chroot=chroot,
grub_timeout=CONF.timeout)
gu.grub1_install(install_devices, boot_device, chroot=chroot)
else:
# TODO(kozhukalov): implement which kernel to use by default
# Currently only grub1_cfg accepts kernel and initrd
# parameters.
gu.grub2_cfg(kernel_params=grub.kernel_params, chroot=chroot,
grub_timeout=CONF.timeout)
gu.grub2_install(install_devices, chroot=chroot)
# TODO(agordeev): move to separate actions?
if CONF.fix_udev_net_rules:
# FIXME(agordeev) There's no convenient way to perfrom NIC
# remapping in Ubuntu, so injecting files prior the first boot
# should work
with open(chroot + '/etc/udev/rules.d/70-persistent-net.rules',
'wt', encoding='utf-8') as f:
f.write(u'# Generated by bareon during provisioning: '
u'BEGIN\n')
# pattern is aa:bb:cc:dd:ee:ff_eth0,aa:bb:cc:dd:ee:ff_eth1
for mapping in self.driver.configdrive_scheme. \
common.udevrules.split(','):
mac_addr, nic_name = mapping.split('_')
f.write(u'SUBSYSTEM=="net", ACTION=="add", '
u'DRIVERS=="?*", ATTR{address}=="%s", '
u'ATTR{type}=="1", KERNEL=="eth*", '
u'NAME="%s"\n' % (mac_addr, nic_name))
f.write(
u'# Generated by bareon during provisioning: END\n')
# FIXME(agordeev): Disable net-generator that adds new entries
# to 70-persistent-net.rules
with open(chroot + '/etc/udev/rules.d/'
'75-persistent-net-generator.rules', 'wt',
encoding='utf-8') as f:
f.write(u'# Generated by bareon during provisioning:\n'
u'# DO NOT DELETE. It is needed to disable '
u'net-generator\n')
# FIXME(kozhukalov): Prevent nailgun-agent from doing anything.
# This ugly hack is to be used together with the command removing
# this lock file not earlier than /etc/rc.local
# The reason for this hack to appear is to prevent nailgun-agent
# from changing mcollective config at the same time when cloud-init
# does the same. Otherwise, we can end up with corrupted
# mcollective config.
# For details see https://bugs.launchpad.net/fuel/+bug/1449186
LOG.debug('Preventing nailgun-agent from doing '
'anything until it is unlocked')
utils.makedirs_if_not_exists(os.path.join(chroot,
'etc/nailgun-agent'))
with open(os.path.join(chroot, 'etc/nailgun-agent/nodiscover'),
'w'):
pass
with open(chroot + '/etc/fstab', 'wt', encoding='utf-8') as f:
for fs in self.driver.partition_scheme.fss:
# TODO(kozhukalov): Think of improving the logic so as to
# insert a meaningful fsck order value which is last zero
# at fstab line. Currently we set it into 0 which means
# a corresponding file system will never be checked. We
# assume puppet or other configuration tool will care of
# it.
if fs.mount == '/':
f.write(u'UUID=%s %s %s defaults,errors=panic 0 0\n' %
(mount2uuid[fs.mount], fs.mount, fs.type))
else:
f.write(u'UUID=%s %s %s defaults 0 0\n' %
(mount2uuid[fs.mount], fs.mount, fs.type))
def do_mkbootstrap(self):
"""Building bootstrap image
Currently supports only Ubuntu-Trusty
Includes the following steps
1) Allocate and configure debootstrap.
2) Install packages
3) Run user-post script(is defined)
4) populate squashfs\init\vmlinuz files
5) create metadata.yaml and pack thats all into tar.gz
"""
LOG.info('--- Building bootstrap image (do_mkbootstrap) ---')
driver_os = self.driver.operating_system
# c_dir = output container directory, where all builded files will
# be stored, before packaging into archive
LOG.debug('Creating bootstrap container folder')
c_dir = bu.mkdtemp_smart(CONF.image_build_dir,
CONF.image_build_suffix + '_container')
try:
chroot = bu.mkdtemp_smart(
CONF.image_build_dir, CONF.image_build_suffix)
self.install_base_os(chroot)
bs_scheme = self.driver.bootstrap_scheme
# init modules, needed for bootstrap. Currently
# we support only one scheme initrd + rootfs + kernel
initrd = filter(lambda x: x.name == 'initrd',
bs_scheme.modules)[0]
rootfs = filter(lambda x: x.name == 'rootfs',
bs_scheme.modules)[0]
metadata = {}
metadata['os'] = driver_os.to_dict()
packages = driver_os.packages
metadata['packages'] = packages
self._set_apt_repos(
chroot, driver_os.repos,
proxies=driver_os.proxies.proxies,
direct_repo_addrs=driver_os.proxies.direct_repo_addr_list)
self._update_metadata_with_repos(
metadata, driver_os.repos)
LOG.debug('Installing packages using apt-get: %s',
' '.join(packages))
# disable hosts/resolv files
bu.propagate_host_resolv_conf(chroot)
if hasattr(bs_scheme, 'certs') and bs_scheme.certs:
bu.copy_update_certs(bs_scheme.certs, chroot)
bu.run_apt_get(chroot, packages=packages,
attempts=CONF.fetch_packages_attempts)
LOG.debug('Post-install OS configuration')
if hasattr(bs_scheme, 'extra_files') and bs_scheme.extra_files:
for extra in bs_scheme.extra_files:
bu.rsync_inject(extra, chroot)
if (hasattr(bs_scheme, 'root_ssh_authorized_file') and
bs_scheme.root_ssh_authorized_file):
LOG.debug('Put ssh auth file %s',
bs_scheme.root_ssh_authorized_file)
auth_file = os.path.join(chroot, 'root/.ssh/authorized_keys')
utils.makedirs_if_not_exists(os.path.dirname(
auth_file), mode=0o700)
shutil.copy(
bs_scheme.root_ssh_authorized_file,
auth_file)
os.chmod(auth_file, 0o700)
# Allow user to drop and run script inside chroot:
if (hasattr(bs_scheme, 'post_script_file') and
bs_scheme.post_script_file):
bu.run_script_in_chroot(
chroot, bs_scheme.post_script_file)
# Save runtime_uuid into bootstrap
bu.dump_runtime_uuid(bs_scheme.uuid,
os.path.join(chroot,
'etc/nailgun-agent/config.yaml'))
# NOTE(sslypushenko) Preferred names in LVM config should updated
# due to point LVM to work only with /dev/mapper folder
bu.override_lvm_config(
chroot,
{'devices': {
'preferred_names': CONF.mpath_lvm_preferred_names}},
lvm_conf_path=CONF.lvm_conf_path)
root = driver_os.get_user_by_name('root')
bu.do_post_inst(chroot,
hashed_root_password=root.hashed_password,
allow_unsigned_file=CONF.allow_unsigned_file,
force_ipv4_file=CONF.force_ipv4_file)
# restore disabled hosts/resolv files
bu.restore_resolv_conf(chroot)
metadata['all_packages'] = bu.get_installed_packages(chroot)
# We need to recompress initramfs with new compression:
bu.recompress_initramfs(
chroot,
compress=initrd.compress_format)
# Bootstrap nodes load the kernel and initramfs via the network,
# therefore remove the kernel and initramfs located in root
# filesystem to make the image smaller (and save the network
# bandwidth and the boot time)
bu.copy_kernel_initramfs(chroot, c_dir, clean=True)
LOG.debug('Making sure there are no running processes '
'inside chroot before trying to umount chroot')
if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
if not bu.stop_chrooted_processes(
chroot, signal=signal.SIGKILL):
raise errors.UnexpectedProcessError(
'Stopping chrooted processes failed. '
'There are some processes running in chroot %s',
chroot)
bu.run_mksquashfs(
chroot, os.path.join(c_dir, os.path.basename(rootfs.uri)),
rootfs.compress_format)
self.dump_mkbootstrap_meta(metadata, c_dir, bs_scheme)
output = bu.save_bs_container(self.driver.output, c_dir,
bs_scheme.container.format)
LOG.info('--- Building bootstrap image END (do_mkbootstrap) ---')
return output
except Exception as exc:
LOG.error('Failed to build bootstrap image: %s', exc)
raise
finally:
LOG.info('Cleanup chroot')
self.destroy_chroot(chroot)
try:
shutil.rmtree(c_dir)
except OSError:
LOG.debug('Finally: directory %s seems does not exist '
'or can not be removed', c_dir)
def install_base_os(self, chroot):
"""Bootstrap a basic Linux system
:param chroot directory where the installed OS can be found
For now only Ubuntu is supported.
Note: the data gets written to a different location (a set of
ext4 images located in the image_build_dir directory)
Includes the following steps
1) create temporary sparse files for all images (truncate)
2) attach temporary files to loop devices (losetup)
3) create file systems on these loop devices
4) create temporary chroot directory
5) mount loop devices into chroot directory
6) install operating system (debootstrap and apt-get)
"""
LOG.info('*** Preparing image space ***')
for image in self.driver.image_scheme.images:
LOG.debug('Creating temporary sparsed file for the '
'image: %s', image.uri)
img_tmp_file = bu.create_sparse_tmp_file(
dir=CONF.image_build_dir, suffix=CONF.image_build_suffix,
size=CONF.sparse_file_size)
LOG.debug('Temporary file: %s', img_tmp_file)
# we need to remember those files
# to be able to shrink them and move in the end
image.img_tmp_file = img_tmp_file
image.target_device.name = \
bu.attach_file_to_free_loop_device(
img_tmp_file,
max_loop_devices_count=CONF.max_loop_devices_count,
loop_device_major_number=CONF.loop_device_major_number,
max_attempts=CONF.max_allowed_attempts_attach_image)
# find fs with the same loop device object
# as image.target_device
fs = self.driver.partition_scheme.fs_by_device(
image.target_device)
LOG.debug('Creating file system on the image')
fu.make_fs(
fs_type=fs.type,
fs_options=fs.options,
fs_label=fs.label,
dev=six.text_type(fs.device))
if fs.type == 'ext4':
LOG.debug('Trying to disable journaling for ext4 '
'in order to speed up the build')
utils.execute('tune2fs', '-O', '^has_journal',
six.text_type(fs.device))
# mounting all images into chroot tree
self.mount_target(chroot, treat_mtab=False, pseudo=False)
LOG.info('Installing BASE operating system into image')
# FIXME(kozhukalov): !!! we need this part to be OS agnostic
# DEBOOTSTRAP
# we use first repo as the main mirror
uri = self.driver.operating_system.repos[0].uri
suite = self.driver.operating_system.repos[0].suite
proxies = self.driver.operating_system.proxies
LOG.debug('Preventing services from being get started')
bu.suppress_services_start(chroot)
LOG.debug('Installing base operating system using debootstrap')
bu.run_debootstrap(uri=uri, suite=suite, chroot=chroot,
attempts=CONF.fetch_packages_attempts,
proxies=proxies.proxies,
direct_repo_addr=proxies.direct_repo_addr_list)
# APT-GET
LOG.debug('Configuring apt inside chroot')
LOG.debug('Setting environment variables')
bu.set_apt_get_env()
LOG.debug('Allowing unauthenticated repos')
bu.pre_apt_get(chroot,
allow_unsigned_file=CONF.allow_unsigned_file,
force_ipv4_file=CONF.force_ipv4_file,
proxies=proxies.proxies,
direct_repo_addr=proxies.direct_repo_addr_list)
# we need /proc to be mounted for apt-get success
LOG.debug('Preventing services from being get started')
bu.suppress_services_start(chroot)
utils.makedirs_if_not_exists(os.path.join(chroot, 'proc'))
# we need /proc to be mounted for apt-get success
fu.mount_bind(chroot, '/proc')
bu.populate_basic_dev(chroot)
def test_makedirs_if_not_exists(self, mock_isdir, mock_makedirs):
utils.makedirs_if_not_exists('/fake/path')
mock_isdir.assert_called_once_with('/fake/path')
mock_makedirs.assert_called_once_with('/fake/path', mode=0o755)
def test_makedirs_if_not_exists_already_exists(self, mock_isdir, mock_makedirs):
utils.makedirs_if_not_exists("/fake/path")
mock_isdir.assert_called_once_with("/fake/path")
self.assertEqual(mock_makedirs.mock_calls, [])
def test_makedirs_if_not_exists_mode_given(self, mock_isdir, mock_makedirs):
utils.makedirs_if_not_exists("/fake/path", mode=0o000)
mock_isdir.assert_called_once_with("/fake/path")
mock_makedirs.assert_called_once_with("/fake/path", mode=0o000)
def test_makedirs_if_not_exists_already_exists(self, mock_isdir,
mock_makedirs):
utils.makedirs_if_not_exists('/fake/path')
mock_isdir.assert_called_once_with('/fake/path')
self.assertEqual(mock_makedirs.mock_calls, [])
def test_makedirs_if_not_exists_mode_given(self, mock_isdir,
mock_makedirs):
utils.makedirs_if_not_exists('/fake/path', mode=0o000)
mock_isdir.assert_called_once_with('/fake/path')
mock_makedirs.assert_called_once_with('/fake/path', mode=0o000)
def install_base_os(self, chroot):
"""Bootstrap a basic Linux system
:param chroot directory where the installed OS can be found
For now only Ubuntu is supported.
Note: the data gets written to a different location (a set of
ext4 images located in the image_build_dir directory)
Includes the following steps
1) create temporary sparse files for all images (truncate)
2) attach temporary files to loop devices (losetup)
3) create file systems on these loop devices
4) create temporary chroot directory
5) mount loop devices into chroot directory
6) install operating system (debootstrap and apt-get)
"""
LOG.info('*** Preparing image space ***')
for image in self.driver.image_scheme.images:
LOG.debug('Creating temporary sparsed file for the '
'image: %s', image.uri)
img_tmp_file = bu.create_sparse_tmp_file(
dir=CONF.image_build_dir,
suffix=CONF.image_build_suffix,
size=CONF.sparse_file_size)
LOG.debug('Temporary file: %s', img_tmp_file)
# we need to remember those files
# to be able to shrink them and move in the end
image.img_tmp_file = img_tmp_file
image.target_device.name = \
bu.attach_file_to_free_loop_device(
img_tmp_file,
max_loop_devices_count=CONF.max_loop_devices_count,
loop_device_major_number=CONF.loop_device_major_number,
max_attempts=CONF.max_allowed_attempts_attach_image)
# find fs with the same loop device object
# as image.target_device
fs = self.driver.partition_scheme.fs_by_device(image.target_device)
LOG.debug('Creating file system on the image')
fu.make_fs(fs_type=fs.type,
fs_options=fs.options,
fs_label=fs.label,
dev=six.text_type(fs.device))
if fs.type == 'ext4':
LOG.debug('Trying to disable journaling for ext4 '
'in order to speed up the build')
utils.execute('tune2fs', '-O', '^has_journal',
six.text_type(fs.device))
# mounting all images into chroot tree
self.mount_target(chroot, treat_mtab=False, pseudo=False)
LOG.info('Installing BASE operating system into image')
# FIXME(kozhukalov): !!! we need this part to be OS agnostic
# DEBOOTSTRAP
# we use first repo as the main mirror
uri = self.driver.operating_system.repos[0].uri
suite = self.driver.operating_system.repos[0].suite
proxies = self.driver.operating_system.proxies
LOG.debug('Preventing services from being get started')
bu.suppress_services_start(chroot)
LOG.debug('Installing base operating system using debootstrap')
bu.run_debootstrap(uri=uri,
suite=suite,
chroot=chroot,
attempts=CONF.fetch_packages_attempts,
proxies=proxies.proxies,
direct_repo_addr=proxies.direct_repo_addr_list)
# APT-GET
LOG.debug('Configuring apt inside chroot')
LOG.debug('Setting environment variables')
bu.set_apt_get_env()
LOG.debug('Allowing unauthenticated repos')
bu.pre_apt_get(chroot,
allow_unsigned_file=CONF.allow_unsigned_file,
force_ipv4_file=CONF.force_ipv4_file,
proxies=proxies.proxies,
direct_repo_addr=proxies.direct_repo_addr_list)
# we need /proc to be mounted for apt-get success
LOG.debug('Preventing services from being get started')
bu.suppress_services_start(chroot)
utils.makedirs_if_not_exists(os.path.join(chroot, 'proc'))
# we need /proc to be mounted for apt-get success
fu.mount_bind(chroot, '/proc')
bu.populate_basic_dev(chroot)
def do_mkbootstrap(self):
"""Building bootstrap image
Currently supports only Ubuntu-Trusty
Includes the following steps
1) Allocate and configure debootstrap.
2) Install packages
3) Run user-post script(is defined)
4) populate squashfs\init\vmlinuz files
5) create metadata.yaml and pack thats all into tar.gz
"""
LOG.info('--- Building bootstrap image (do_mkbootstrap) ---')
driver_os = self.driver.operating_system
# c_dir = output container directory, where all builded files will
# be stored, before packaging into archive
LOG.debug('Creating bootstrap container folder')
c_dir = bu.mkdtemp_smart(CONF.image_build_dir,
CONF.image_build_suffix + '_container')
try:
chroot = bu.mkdtemp_smart(CONF.image_build_dir,
CONF.image_build_suffix)
self.install_base_os(chroot)
bs_scheme = self.driver.bootstrap_scheme
# init modules, needed for bootstrap. Currently
# we support only one scheme initrd + rootfs + kernel
initrd = filter(lambda x: x.name == 'initrd', bs_scheme.modules)[0]
rootfs = filter(lambda x: x.name == 'rootfs', bs_scheme.modules)[0]
metadata = {}
metadata['os'] = driver_os.to_dict()
packages = driver_os.packages
metadata['packages'] = packages
self._set_apt_repos(
chroot,
driver_os.repos,
proxies=driver_os.proxies.proxies,
direct_repo_addrs=driver_os.proxies.direct_repo_addr_list)
self._update_metadata_with_repos(metadata, driver_os.repos)
LOG.debug('Installing packages using apt-get: %s',
' '.join(packages))
# disable hosts/resolv files
bu.propagate_host_resolv_conf(chroot)
if hasattr(bs_scheme, 'certs') and bs_scheme.certs:
bu.copy_update_certs(bs_scheme.certs, chroot)
bu.run_apt_get(chroot,
packages=packages,
attempts=CONF.fetch_packages_attempts)
LOG.debug('Post-install OS configuration')
if hasattr(bs_scheme, 'extra_files') and bs_scheme.extra_files:
for extra in bs_scheme.extra_files:
bu.rsync_inject(extra, chroot)
if (hasattr(bs_scheme, 'root_ssh_authorized_file')
and bs_scheme.root_ssh_authorized_file):
LOG.debug('Put ssh auth file %s',
bs_scheme.root_ssh_authorized_file)
auth_file = os.path.join(chroot, 'root/.ssh/authorized_keys')
utils.makedirs_if_not_exists(os.path.dirname(auth_file),
mode=0o700)
shutil.copy(bs_scheme.root_ssh_authorized_file, auth_file)
os.chmod(auth_file, 0o700)
# Allow user to drop and run script inside chroot:
if (hasattr(bs_scheme, 'post_script_file')
and bs_scheme.post_script_file):
bu.run_script_in_chroot(chroot, bs_scheme.post_script_file)
# Save runtime_uuid into bootstrap
bu.dump_runtime_uuid(
bs_scheme.uuid,
os.path.join(chroot, 'etc/nailgun-agent/config.yaml'))
bu.do_post_inst(chroot,
allow_unsigned_file=CONF.allow_unsigned_file,
force_ipv4_file=CONF.force_ipv4_file)
# restore disabled hosts/resolv files
bu.restore_resolv_conf(chroot)
metadata['all_packages'] = bu.get_installed_packages(chroot)
# We need to recompress initramfs with new compression:
bu.recompress_initramfs(chroot, compress=initrd.compress_format)
# Bootstrap nodes load the kernel and initramfs via the network,
# therefore remove the kernel and initramfs located in root
# filesystem to make the image smaller (and save the network
# bandwidth and the boot time)
bu.copy_kernel_initramfs(chroot, c_dir, clean=True)
LOG.debug('Making sure there are no running processes '
'inside chroot before trying to umount chroot')
if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
if not bu.stop_chrooted_processes(chroot,
signal=signal.SIGKILL):
raise errors.UnexpectedProcessError(
'Stopping chrooted processes failed. '
'There are some processes running in chroot %s',
chroot)
bu.run_mksquashfs(
chroot, os.path.join(c_dir, os.path.basename(rootfs.uri)),
rootfs.compress_format)
self.dump_mkbootstrap_meta(metadata, c_dir, bs_scheme)
output = bu.save_bs_container(self.driver.output, c_dir,
bs_scheme.container.format)
LOG.info('--- Building bootstrap image END (do_mkbootstrap) ---')
return output
except Exception as exc:
LOG.error('Failed to build bootstrap image: %s', exc)
raise
finally:
LOG.info('Cleanup chroot')
self.destroy_chroot(chroot)
try:
shutil.rmtree(c_dir)
except OSError:
LOG.debug(
'Finally: directory %s seems does not exist '
'or can not be removed', c_dir)
