def save_bs_container(output, input_dir, format="tar.gz"):
"""Copy files from dir to archive or another directory
:param output:
:param input_dir:
:param format:
:return:
"""
if format == 'directory':
utils.makedirs_if_not_exists(output)
bs_files = os.listdir(input_dir)
LOG.debug("Output folder: %s\ntry to copy bootstrap files: %s", output,
bs_files)
for bs_file in bs_files:
abs_bs_file = os.path.join(input_dir, bs_file)
if (os.path.isfile(abs_bs_file)):
if os.path.isfile(os.path.join(output, bs_file)):
raise errors.BootstrapFileAlreadyExists(
"File: {0} already exists in: {1}".format(
bs_file, output))
shutil.copy(abs_bs_file, output)
os.chmod(os.path.join(output, bs_file), 0o755)
return output
elif format == 'tar.gz':
LOG.debug("Try to make output archive file: %s", output)
output = make_targz(input_dir, output_name=output)
return output
else:
raise errors.WrongOutputContainer(
"Unsupported bootstrap container format {0}.".format(format))
def mount_target(self, chroot, treat_mtab=True, pseudo=True):
"""Mount a set of file systems into a chroot
:param chroot: Directory where to mount file systems
:param treat_mtab: If mtab needs to be actualized (Default: True)
:param pseudo: If pseudo file systems
need to be mounted (Default: True)
"""
LOG.debug("Mounting target file systems: %s", chroot)
# Here we are going to mount all file systems in partition scheme.
for fs in self.driver.partition_scheme.fs_sorted_by_depth():
if fs.mount == "swap":
continue
mount = chroot + fs.mount
utils.makedirs_if_not_exists(mount)
fu.mount_fs(fs.type, str(fs.device), mount)
if pseudo:
for path in ("/sys", "/dev", "/proc"):
utils.makedirs_if_not_exists(chroot + path)
fu.mount_bind(chroot, path)
if treat_mtab:
mtab = utils.execute("chroot", chroot, "grep", "-v", "rootfs", "/proc/mounts")[0]
mtab_path = chroot + "/etc/mtab"
if os.path.islink(mtab_path):
os.remove(mtab_path)
with open(mtab_path, "wt", encoding="utf-8") as f:
f.write(six.text_type(mtab))
def mount_target(self, chroot, treat_mtab=True, pseudo=True):
"""Mount a set of file systems into a chroot
:param chroot: Directory where to mount file systems
:param treat_mtab: If mtab needs to be actualized (Default: True)
:param pseudo: If pseudo file systems
need to be mounted (Default: True)
"""
LOG.debug('Mounting target file systems: %s', chroot)
# Here we are going to mount all file systems in partition scheme.
for fs in self.driver.partition_scheme.fs_sorted_by_depth():
if fs.mount == 'swap':
continue
mount = chroot + fs.mount
utils.makedirs_if_not_exists(mount)
fu.mount_fs(fs.type, str(fs.device), mount)
if pseudo:
for path in ('/sys', '/dev', '/proc'):
utils.makedirs_if_not_exists(chroot + path)
fu.mount_bind(chroot, path)
if treat_mtab:
mtab = utils.execute('chroot', chroot, 'grep', '-v', 'rootfs',
'/proc/mounts')[0]
mtab_path = chroot + '/etc/mtab'
if os.path.islink(mtab_path):
os.remove(mtab_path)
with open(mtab_path, 'wb') as f:
f.write(mtab)
def copy_kernel_initramfs(chroot, dstdir, clean=False):
"""Copy latest or newest vmlinuz and initrd from chroot
:param chroot:
:param dstdir: copy to folder
:param clean: remove all vmlinuz\initrd after done
:return:
"""
# TODO(azvyagintsev) fetch from uri driver
# module* : result filename
files = {'vmlinuz': 'vmlinuz', 'initrd': 'initrd.img'}
utils.makedirs_if_not_exists(dstdir)
boot_dir = os.path.join(chroot, 'boot')
for module in six.iterkeys(files):
mask = os.path.join(boot_dir, module + '*')
all_files = glob.glob(mask)
if len(all_files) > 1:
raise errors.TooManyKernels("Too many %s detected :%s", module,
all_files)
file_to_copy = all_files[0]
copy_to = os.path.join(dstdir, files[module])
LOG.debug('Copying file: %s to: %s', file_to_copy, copy_to)
shutil.copy(file_to_copy, copy_to)
if clean:
files_to_remove = glob.glob(mask)
remove_files('/', files_to_remove)
def copy_kernel_initramfs(chroot, dstdir, clean=False):
"""Copy latest or newest vmlinuz and initrd from chroot
:param chroot:
:param dstdir: copy to folder
:param clean: remove all vmlinuz\initrd after done
:return:
"""
# TODO(azvyagintsev) fetch from uri driver
# module* : result filename
files = {'vmlinuz': 'vmlinuz',
'initrd': 'initrd.img'
}
utils.makedirs_if_not_exists(dstdir)
boot_dir = os.path.join(chroot, 'boot')
for module in six.iterkeys(files):
mask = os.path.join(boot_dir, module + '*')
all_files = glob.glob(mask)
if len(all_files) > 1:
raise errors.TooManyKernels(
"Too many %s detected :%s", module, all_files)
file_to_copy = all_files[0]
copy_to = os.path.join(dstdir, files[module])
LOG.debug('Copying file: %s to: %s', file_to_copy, copy_to)
shutil.copy(file_to_copy, copy_to)
if clean:
files_to_remove = glob.glob(mask)
remove_files('/', files_to_remove)
def mount_target(self, chroot, treat_mtab=True, pseudo=True):
"""Mount a set of file systems into a chroot
:param chroot: Directory where to mount file systems
:param treat_mtab: If mtab needs to be actualized (Default: True)
:param pseudo: If pseudo file systems
need to be mounted (Default: True)
"""
LOG.debug('Mounting target file systems: %s', chroot)
# Here we are going to mount all file systems in partition scheme.
for fs in self.driver.partition_scheme.fs_sorted_by_depth():
if fs.mount == 'swap':
continue
mount = chroot + fs.mount
utils.makedirs_if_not_exists(mount)
fu.mount_fs(fs.type, str(fs.device), mount)
if pseudo:
for path in ('/sys', '/dev', '/proc'):
utils.makedirs_if_not_exists(chroot + path)
fu.mount_bind(chroot, path)
if treat_mtab:
mtab = utils.execute(
'chroot', chroot, 'grep', '-v', 'rootfs', '/proc/mounts')[0]
mtab_path = chroot + '/etc/mtab'
if os.path.islink(mtab_path):
os.remove(mtab_path)
with open(mtab_path, 'wt', encoding='utf-8') as f:
f.write(six.text_type(mtab))
def save_bs_container(output, input_dir, format="tar.gz"):
"""Copy files from dir to archive or another directory
:param output:
:param input_dir:
:param format:
:return:
"""
if format == 'directory':
utils.makedirs_if_not_exists(output)
bs_files = os.listdir(input_dir)
LOG.debug("Output folder: %s\ntry to copy bootstrap files: %s",
output, bs_files)
for bs_file in bs_files:
abs_bs_file = os.path.join(input_dir, bs_file)
if (os.path.isfile(abs_bs_file)):
if os.path.isfile(os.path.join(output, bs_file)):
raise errors.BootstrapFileAlreadyExists(
"File: {0} already exists in: {1}"
.format(bs_file, output))
shutil.copy(abs_bs_file, output)
os.chmod(os.path.join(output, bs_file), 0o755)
return output
elif format == 'tar.gz':
LOG.debug("Try to make output archive file: %s", output)
output = make_targz(input_dir, output_name=output)
return output
else:
raise errors.WrongOutputContainer(
"Unsupported bootstrap container format {0}."
.format(format))
def do_post_inst(chroot,
hashed_root_password,
allow_unsigned_file='allow_unsigned_packages',
force_ipv4_file='force_ipv4',
pipeline_depth_file='pipeline_depth',
install_rule_file='install_rule'):
# NOTE(agordeev): set up password for root
utils.execute('sed', '-i',
's%root:[\*,\!]%root:' + hashed_root_password + '%',
os.path.join(chroot, 'etc/shadow'))
# NOTE(agordeev): backport from bash-script:
# in order to prevent the later puppet workflow outage, puppet service
# should be disabled on a node startup.
# Being enabled by default, sometimes it leads to puppet service hanging
# and recognizing the deployment as failed.
# TODO(agordeev): take care of puppet service for other distros, once
# fuel-agent will be capable of building images for them too.
if os.path.exists(os.path.join(chroot, 'etc/init.d/puppet')):
utils.execute('chroot', chroot, 'update-rc.d', 'puppet', 'disable')
# NOTE(agordeev): disable mcollective to be automatically started on boot
# to prevent confusing messages in its log (regarding connection errors).
with open(os.path.join(chroot, 'etc/init/mcollective.override'), 'w') as f:
f.write("manual\n")
service_link = os.path.join(
chroot,
'etc/systemd/system/multi-user.target.wants/mcollective.service')
if os.path.exists(service_link):
os.unlink(service_link)
cloud_cfg = os.path.join(chroot, 'etc/cloud/cloud.cfg.d/')
utils.makedirs_if_not_exists(os.path.dirname(cloud_cfg))
with open(
os.path.join(
chroot, 'etc/cloud/cloud.cfg.d/99-disable-network-config.cfg'),
'w') as cf:
yaml.safe_dump({'network': {
'config': 'disabled'
}},
cf,
encoding='utf-8',
default_flow_style=False)
cloud_init_conf = os.path.join(chroot, 'etc/cloud/cloud.cfg')
if os.path.exists(cloud_init_conf):
fix_cloud_init_config(cloud_init_conf)
# NOTE(agordeev): remove custom policy-rc.d which is needed to disable
# execution of post/pre-install package hooks and start of services
remove_files(chroot, ['usr/sbin/policy-rc.d'])
# enable mdadm (remove nomdadmddf nomdadmism options from cmdline)
utils.execute('chroot', chroot, 'dpkg-divert', '--local', '--add',
os.path.join('/', GRUB2_DMRAID_SETTINGS))
remove_files(chroot, [GRUB2_DMRAID_SETTINGS])
# remove cached apt files
utils.execute('chroot', chroot, 'apt-get', 'clean')
clean_apt_settings(chroot,
allow_unsigned_file=allow_unsigned_file,
force_ipv4_file=force_ipv4_file,
pipeline_depth_file=pipeline_depth_file,
install_rule_file=install_rule_file)
def restore_resolv_conf(chroot):
"""Restore hosts/resolv files in chroot
opposite to propagate_host_resolv_conf
"""
c_etc = os.path.join(chroot, 'etc/')
utils.makedirs_if_not_exists(c_etc)
for conf_name in ('resolv.conf', 'hosts'):
dst_conf_name = os.path.join(c_etc, conf_name)
if os.path.isfile(dst_conf_name + '.bak'):
LOG.debug('Restoring default %s inside chroot', conf_name)
shutil.move(dst_conf_name + '.bak', dst_conf_name)
def restore_resolv_conf(chroot):
"""Restore hosts/resolv files in chroot
opposite to propagate_host_resolv_conf
"""
c_etc = os.path.join(chroot, 'etc/')
utils.makedirs_if_not_exists(c_etc)
for conf_name in ('resolv.conf', 'hosts'):
dst_conf_name = os.path.join(c_etc, conf_name)
if os.path.isfile(dst_conf_name + '.bak'):
LOG.debug('Restoring default %s inside chroot', conf_name)
shutil.move(dst_conf_name + '.bak', dst_conf_name)
def mkdtemp_smart(root_dir, suffix):
"""Create a unique temporary directory in root_dir
Automatically creates root_dir if it does not exist.
Otherwise same as tempfile.mkdtemp
"""
LOG.debug('Creating temporary chroot directory')
utils.makedirs_if_not_exists(root_dir)
chroot = tempfile.mkdtemp(dir=root_dir, suffix=suffix)
LOG.debug('Temporary chroot dir: %s', chroot)
return chroot
def mkdtemp_smart(root_dir, suffix):
"""Create a unique temporary directory in root_dir
Automatically creates root_dir if it does not exist.
Otherwise same as tempfile.mkdtemp
"""
LOG.debug('Creating temporary chroot directory')
utils.makedirs_if_not_exists(root_dir)
chroot = tempfile.mkdtemp(
dir=root_dir, suffix=suffix)
LOG.debug('Temporary chroot dir: %s', chroot)
return chroot
def make_targz(source_dir, output_name=None):
"""Archive the given directory
:param source_dir: directory to archive
:param output_name: output file name, might be a relative
or an absolute path
"""
if not output_name:
output_name = six.text_type(uuid.uuid4()) + '.tar.gz'
utils.makedirs_if_not_exists(os.path.dirname(output_name))
LOG.info('Creating archive: %s', output_name)
utils.execute('tar', '-czf', output_name, '--directory',
os.path.normcase(source_dir), '.', logged=True)
return output_name
def do_post_inst(chroot, hashed_root_password,
allow_unsigned_file='allow_unsigned_packages',
force_ipv4_file='force_ipv4',
pipeline_depth_file='pipeline_depth'):
# NOTE(agordeev): set up password for root
utils.execute('sed', '-i',
's%root:[\*,\!]%root:' + hashed_root_password + '%',
os.path.join(chroot, 'etc/shadow'))
# NOTE(agordeev): backport from bash-script:
# in order to prevent the later puppet workflow outage, puppet service
# should be disabled on a node startup.
# Being enabled by default, sometimes it leads to puppet service hanging
# and recognizing the deployment as failed.
# TODO(agordeev): take care of puppet service for other distros, once
# fuel-agent will be capable of building images for them too.
if os.path.exists(os.path.join(chroot, 'etc/init.d/puppet')):
utils.execute('chroot', chroot, 'update-rc.d', 'puppet', 'disable')
# NOTE(agordeev): disable mcollective to be automatically started on boot
# to prevent confusing messages in its log (regarding connection errors).
with open(os.path.join(chroot, 'etc/init/mcollective.override'), 'w') as f:
f.write("manual\n")
if os.path.exists(os.path.join(chroot, 'etc/systemd/system')):
os.symlink('/dev/null', os.path.join(chroot,
'etc/systemd/system/mcollective.service'))
cloud_cfg = os.path.join(chroot, 'etc/cloud/cloud.cfg.d/')
utils.makedirs_if_not_exists(os.path.dirname(cloud_cfg))
with open(os.path.join(
chroot,
'etc/cloud/cloud.cfg.d/99-disable-network-config.cfg'), 'w') as cf:
yaml.safe_dump({'network': {'config': 'disabled'}}, cf,
encoding='utf-8',
default_flow_style=False)
cloud_init_conf = os.path.join(chroot, 'etc/cloud/cloud.cfg')
if os.path.exists(cloud_init_conf):
fix_cloud_init_config(cloud_init_conf)
# NOTE(agordeev): remove custom policy-rc.d which is needed to disable
# execution of post/pre-install package hooks and start of services
remove_files(chroot, ['usr/sbin/policy-rc.d'])
# enable mdadm (remove nomdadmddf nomdadmism options from cmdline)
utils.execute('chroot', chroot, 'dpkg-divert', '--local', '--add',
os.path.join('/', GRUB2_DMRAID_SETTINGS))
remove_files(chroot, [GRUB2_DMRAID_SETTINGS])
# remove cached apt files
utils.execute('chroot', chroot, 'apt-get', 'clean')
clean_apt_settings(chroot, allow_unsigned_file=allow_unsigned_file,
force_ipv4_file=force_ipv4_file,
pipeline_depth_file=pipeline_depth_file)
def configure_admin_nic_ubuntu(chroot, iface, ip, netmask, gw):
ifaces_dir = '/etc/network/interfaces.d'
ifcfg_path = os.path.join(ifaces_dir, 'ifcfg-{0}'.format(iface))
utils.makedirs_if_not_exists(ifaces_dir)
with open(chroot + '/etc/network/interfaces', 'wt', encoding='utf-8') as f:
f.write(u'# Generated by fuel-agent during provisioning:\n'
u'source-directory /etc/network/interfaces.d\n')
with open(chroot + ifcfg_path, 'wt', encoding='utf-8') as f:
f.write(u'# Generated by fuel-agent during provisioning:\n'
u'auto {iface}\n'
u'iface {iface} inet static\n'
u'\taddress {ip}\n'
u'\tnetmask {netmask}\n'
u'\tgateway {gw}\n'.format(iface=iface,
ip=ip,
netmask=netmask,
gw=gw))
def configure_admin_nic_ubuntu(chroot, iface, ip, netmask, gw):
ifaces_dir = '/etc/network/interfaces.d'
ifcfg_path = os.path.join(ifaces_dir, 'ifcfg-{0}'.format(iface))
utils.makedirs_if_not_exists(ifaces_dir)
with open(chroot + '/etc/network/interfaces', 'wt', encoding='utf-8') as f:
f.write(u'# Generated by fuel-agent during provisioning:\n'
u'source-directory /etc/network/interfaces.d\n')
with open(chroot + ifcfg_path, 'wt', encoding='utf-8') as f:
f.write(u'# Generated by fuel-agent during provisioning:\n'
u'auto {iface}\n'
u'iface {iface} inet static\n'
u'\taddress {ip}\n'
u'\tnetmask {netmask}\n'
u'\tgateway {gw}\n'.format(iface=iface,
ip=ip,
netmask=netmask,
gw=gw))
def rsync_inject(src, dst):
"""Recursively copy the src to dst using full source paths
Example: suppose the source directory looks like
src/etc/myconfig
src/usr/bin/myscript
rsync_inject('src', '/tmp/chroot')
copies src/etc/myconfig to /tmp/chroot/etc/myconfig,
and src/usr/bin/myscript to /tmp/chroot/usr/bin/myscript,
respectively
"""
utils.makedirs_if_not_exists(os.path.dirname(dst))
LOG.debug('Rsync files from %s to: %s', src, dst)
utils.execute('rsync', '-rlptDKv', src + '/', dst + '/', logged=True)
def propagate_host_resolv_conf(chroot):
"""Copy DNS settings from host system to chroot.
Make a backup of original /etc/resolv.conf and /etc/hosts.
# In case user pass some custom rules in hosts\resolv.conf.
opposite to restore_resolv_conf
"""
c_etc = os.path.join(chroot, 'etc/')
utils.makedirs_if_not_exists(c_etc)
for conf_name in ('resolv.conf', 'hosts'):
dst_conf_name = os.path.join(c_etc, conf_name)
src_conf_name = os.path.join('/etc/', conf_name)
files_to_copy = [(dst_conf_name, dst_conf_name + '.bak'),
(src_conf_name, dst_conf_name)]
for src, dst in files_to_copy:
if os.path.isfile(src):
shutil.copy(src, dst)
def rsync_inject(src, dst):
"""Recursively copy the src to dst using full source paths
Example: suppose the source directory looks like
src/etc/myconfig
src/usr/bin/myscript
rsync_inject('src', '/tmp/chroot')
copies src/etc/myconfig to /tmp/chroot/etc/myconfig,
and src/usr/bin/myscript to /tmp/chroot/usr/bin/myscript,
respectively
"""
utils.makedirs_if_not_exists(os.path.dirname(dst))
LOG.debug('Rsync files from %s to: %s', src, dst)
utils.execute('rsync', '-rlptDKv', src + '/',
dst + '/', logged=True)
def propagate_host_resolv_conf(chroot):
"""Copy DNS settings from host system to chroot.
Make a backup of original /etc/resolv.conf and /etc/hosts.
# In case user pass some custom rules in hosts\resolv.conf.
opposite to restore_resolv_conf
"""
c_etc = os.path.join(chroot, 'etc/')
utils.makedirs_if_not_exists(c_etc)
for conf_name in ('resolv.conf', 'hosts'):
dst_conf_name = os.path.join(c_etc, conf_name)
src_conf_name = os.path.join('/etc/', conf_name)
files_to_copy = [(dst_conf_name, dst_conf_name + '.bak'),
(src_conf_name, dst_conf_name)]
for src, dst in files_to_copy:
if os.path.isfile(src):
shutil.copy(src, dst)
def dump_runtime_uuid(uuid, config):
"""Save runtime_uuid into yaml file
Simple uuid variable to identify bootstrap.
Variable will be hard-coded into config yaml file, in build-time
:param uuid:
:param config: yaml file
:return:
"""
data = {}
utils.makedirs_if_not_exists(os.path.dirname(config))
if os.path.isfile(config):
with open(config, 'r') as f:
data = yaml.load(f)
data['runtime_uuid'] = uuid
LOG.debug('Save runtime_uuid:%s to file: %s', uuid, config)
with open(config, 'wt') as f:
yaml.safe_dump(data, stream=f, encoding='utf-8')
def dump_runtime_uuid(uuid, config):
"""Save runtime_uuid into yaml file
Simple uuid variable to identify bootstrap.
Variable will be hard-coded into config yaml file, in build-time
:param uuid:
:param config: yaml file
:return:
"""
data = {}
utils.makedirs_if_not_exists(os.path.dirname(config))
if os.path.isfile(config):
with open(config, 'r') as f:
data = yaml.load(f)
data['runtime_uuid'] = uuid
LOG.debug('Save runtime_uuid:%s to file: %s', uuid, config)
with open(config, 'wt') as f:
yaml.safe_dump(data, stream=f, encoding='utf-8')
def make_targz(source_dir, output_name=None):
"""Archive the given directory
:param source_dir: directory to archive
:param output_name: output file name, might be a relative
or an absolute path
"""
if not output_name:
output_name = six.text_type(uuid.uuid4()) + '.tar.gz'
utils.makedirs_if_not_exists(os.path.dirname(output_name))
LOG.info('Creating archive: %s', output_name)
utils.execute('tar',
'-czf',
output_name,
'--directory',
os.path.normcase(source_dir),
'.',
logged=True)
return output_name
def run_mksquashfs(chroot, output_name=None, compression_algorithm='xz'):
"""Pack the target system as squashfs using mksquashfs
:param chroot: chroot system, to be squashfs'd
:param output_name: output file name, might be a relative
or an absolute path
The kernel squashfs driver has to match with the user space squasfs tools.
Use the mksquashfs provided by the target distro to achieve this.
(typically the distro maintainers are smart enough to ship the correct
version of mksquashfs)
Use mksquashfs installed in the target system
1)Mount tmpfs under chroot/mnt
2)run mksquashfs inside a chroot
3)move result files to dstdir
"""
if not output_name:
output_name = 'root.squashfs' + six.text_type(uuid.uuid4())
utils.makedirs_if_not_exists(os.path.dirname(output_name))
dstdir = os.path.dirname(output_name)
temp = '.mksquashfs.tmp.' + six.text_type(uuid.uuid4())
s_dst = os.path.join(chroot, 'mnt/dst')
s_src = os.path.join(chroot, 'mnt/src')
try:
fu.mount_fs(
'tmpfs', 'mnt_{0}'.format(temp),
(os.path.join(chroot, 'mnt')),
'rw,nodev,nosuid,noatime,mode=0755,size=4M')
utils.makedirs_if_not_exists(s_src)
utils.makedirs_if_not_exists(s_dst)
# Bind mount the chroot to avoid including various temporary/virtual
# files (/proc, /sys, /dev, and so on) into the image
fu.mount_fs(None, chroot, s_src, opts='bind')
fu.mount_fs(None, None, s_src, 'remount,bind,ro')
fu.mount_fs(None, dstdir, s_dst, opts='bind')
# run mksquashfs
chroot_squash = os.path.join('/mnt/dst/' + temp)
long_squash = os.path.join(chroot, 'mnt/dst/{0}'.format(temp))
LOG.info('Building squashfs')
utils.execute(
'chroot', chroot, 'mksquashfs', '/mnt/src',
chroot_squash,
'-comp', compression_algorithm,
'-no-progress', '-noappend', logged=True)
# move to result name
LOG.debug('Moving file: %s to: %s', long_squash, output_name)
shutil.move(long_squash, output_name)
except Exception as exc:
LOG.error('squashfs_image build failed: %s', exc)
raise
finally:
LOG.info('squashfs_image clean-up')
stop_chrooted_processes(chroot, signal=signal.SIGTERM)
fu.umount_fs(os.path.join(chroot, 'mnt/dst'))
fu.umount_fs(os.path.join(chroot, 'mnt/src'))
fu.umount_fs(os.path.join(chroot, 'mnt'))
def test_makedirs_if_not_exists(self, mock_isdir, mock_makedirs):
utils.makedirs_if_not_exists("/fake/path")
mock_isdir.assert_called_once_with("/fake/path")
mock_makedirs.assert_called_once_with("/fake/path", mode=0o755)
def do_build_image(self):
"""Building OS images
Includes the following steps
1) create temporary sparse files for all images (truncate)
2) attach temporary files to loop devices (losetup)
3) create file systems on these loop devices
4) create temporary chroot directory
5) mount loop devices into chroot directory
6) install operating system (debootstrap and apt-get)
7) configure OS (clean sources.list and preferences, etc.)
8) umount loop devices
9) resize file systems on loop devices
10) shrink temporary sparse files (images)
11) containerize (gzip) temporary sparse files
12) move temporary gzipped files to their final location
"""
LOG.info("--- Building image (do_build_image) ---")
# TODO(kozhukalov): Implement metadata
# as a pluggable data driver to avoid any fixed format.
metadata = {}
metadata["os"] = self.driver.operating_system.to_dict()
# TODO(kozhukalov): implement this using image metadata
# we need to compare list of packages and repos
LOG.info("*** Checking if image exists ***")
if all([os.path.exists(img.uri.split("file://", 1)[1]) for img in self.driver.image_scheme.images]):
LOG.debug("All necessary images are available. " "Nothing needs to be done.")
return
LOG.debug("At least one of the necessary images is unavailable. " "Starting build process.")
try:
LOG.debug("Creating temporary chroot directory")
utils.makedirs_if_not_exists(CONF.image_build_dir)
chroot = tempfile.mkdtemp(dir=CONF.image_build_dir, suffix=CONF.image_build_suffix)
LOG.debug("Temporary chroot: %s", chroot)
proc_path = os.path.join(chroot, "proc")
LOG.info("*** Preparing image space ***")
for image in self.driver.image_scheme.images:
LOG.debug("Creating temporary sparsed file for the " "image: %s", image.uri)
img_tmp_file = bu.create_sparse_tmp_file(
dir=CONF.image_build_dir, suffix=CONF.image_build_suffix, size=CONF.sparse_file_size
)
LOG.debug("Temporary file: %s", img_tmp_file)
# we need to remember those files
# to be able to shrink them and move in the end
image.img_tmp_file = img_tmp_file
image.target_device.name = bu.attach_file_to_free_loop_device(
img_tmp_file,
max_loop_devices_count=CONF.max_loop_devices_count,
loop_device_major_number=CONF.loop_device_major_number,
max_attempts=CONF.max_allowed_attempts_attach_image,
)
# find fs with the same loop device object
# as image.target_device
fs = self.driver.partition_scheme.fs_by_device(image.target_device)
LOG.debug("Creating file system on the image")
fu.make_fs(fs_type=fs.type, fs_options=fs.options, fs_label=fs.label, dev=str(fs.device))
if fs.type == "ext4":
LOG.debug("Trying to disable journaling for ext4 " "in order to speed up the build")
utils.execute("tune2fs", "-O", "^has_journal", str(fs.device))
# mounting all images into chroot tree
self.mount_target(chroot, treat_mtab=False, pseudo=False)
LOG.info("*** Shipping image content ***")
LOG.debug("Installing operating system into image")
# FIXME(kozhukalov): !!! we need this part to be OS agnostic
# DEBOOTSTRAP
# we use first repo as the main mirror
uri = self.driver.operating_system.repos[0].uri
suite = self.driver.operating_system.repos[0].suite
LOG.debug("Preventing services from being get started")
bu.suppress_services_start(chroot)
LOG.debug("Installing base operating system using debootstrap")
bu.run_debootstrap(uri=uri, suite=suite, chroot=chroot, attempts=CONF.fetch_packages_attempts)
# APT-GET
LOG.debug("Configuring apt inside chroot")
LOG.debug("Setting environment variables")
bu.set_apt_get_env()
LOG.debug("Allowing unauthenticated repos")
bu.pre_apt_get(chroot, allow_unsigned_file=CONF.allow_unsigned_file, force_ipv4_file=CONF.force_ipv4_file)
for repo in self.driver.operating_system.repos:
LOG.debug(
"Adding repository source: name={name}, uri={uri},"
"suite={suite}, section={section}".format(
name=repo.name, uri=repo.uri, suite=repo.suite, section=repo.section
)
)
bu.add_apt_source(name=repo.name, uri=repo.uri, suite=repo.suite, section=repo.section, chroot=chroot)
LOG.debug(
"Adding repository preference: "
"name={name}, priority={priority}".format(name=repo.name, priority=repo.priority)
)
if repo.priority is not None:
bu.add_apt_preference(
name=repo.name,
priority=repo.priority,
suite=repo.suite,
section=repo.section,
chroot=chroot,
uri=repo.uri,
)
metadata.setdefault("repos", []).append(
{
"type": "deb",
"name": repo.name,
"uri": repo.uri,
"suite": repo.suite,
"section": repo.section,
"priority": repo.priority,
"meta": repo.meta,
}
)
LOG.debug("Preventing services from being get started")
bu.suppress_services_start(chroot)
packages = self.driver.operating_system.packages
metadata["packages"] = packages
# we need /proc to be mounted for apt-get success
utils.makedirs_if_not_exists(proc_path)
fu.mount_bind(chroot, "/proc")
bu.populate_basic_dev(chroot)
LOG.debug("Installing packages using apt-get: %s", " ".join(packages))
bu.run_apt_get(chroot, packages=packages, attempts=CONF.fetch_packages_attempts)
LOG.debug("Post-install OS configuration")
bu.do_post_inst(chroot, allow_unsigned_file=CONF.allow_unsigned_file, force_ipv4_file=CONF.force_ipv4_file)
LOG.debug("Making sure there are no running processes " "inside chroot before trying to umount chroot")
if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
if not bu.stop_chrooted_processes(chroot, signal=signal.SIGKILL):
raise errors.UnexpectedProcessError(
"Stopping chrooted processes failed. " "There are some processes running in chroot %s", chroot
)
LOG.info("*** Finalizing image space ***")
fu.umount_fs(proc_path)
# umounting all loop devices
self.umount_target(chroot, pseudo=False)
for image in self.driver.image_scheme.images:
# find fs with the same loop device object
# as image.target_device
fs = self.driver.partition_scheme.fs_by_device(image.target_device)
if fs.type == "ext4":
LOG.debug("Trying to re-enable journaling for ext4")
utils.execute("tune2fs", "-O", "has_journal", str(fs.device))
LOG.debug("Deattaching loop device from file: %s", image.img_tmp_file)
bu.deattach_loop(str(image.target_device))
LOG.debug("Shrinking temporary image file: %s", image.img_tmp_file)
bu.shrink_sparse_file(image.img_tmp_file)
raw_size = os.path.getsize(image.img_tmp_file)
raw_md5 = utils.calculate_md5(image.img_tmp_file, raw_size)
LOG.debug("Containerizing temporary image file: %s", image.img_tmp_file)
img_tmp_containerized = bu.containerize(
image.img_tmp_file, image.container, chunk_size=CONF.data_chunk_size
)
img_containerized = image.uri.split("file://", 1)[1]
# NOTE(kozhukalov): implement abstract publisher
LOG.debug("Moving image file to the final location: %s", img_containerized)
shutil.move(img_tmp_containerized, img_containerized)
container_size = os.path.getsize(img_containerized)
container_md5 = utils.calculate_md5(img_containerized, container_size)
metadata.setdefault("images", []).append(
{
"raw_md5": raw_md5,
"raw_size": raw_size,
"raw_name": None,
"container_name": os.path.basename(img_containerized),
"container_md5": container_md5,
"container_size": container_size,
"container": image.container,
"format": image.format,
}
)
# NOTE(kozhukalov): implement abstract publisher
LOG.debug("Image metadata: %s", metadata)
with open(self.driver.metadata_uri.split("file://", 1)[1], "wt", encoding="utf-8") as f:
yaml.safe_dump(metadata, stream=f)
LOG.info("--- Building image END (do_build_image) ---")
except Exception as exc:
LOG.error("Failed to build image: %s", exc)
raise
finally:
LOG.debug("Finally: stopping processes inside chroot: %s", chroot)
if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
bu.stop_chrooted_processes(chroot, signal=signal.SIGKILL)
LOG.debug("Finally: umounting procfs %s", proc_path)
fu.umount_fs(proc_path)
LOG.debug("Finally: umounting chroot tree %s", chroot)
self.umount_target(chroot, pseudo=False)
for image in self.driver.image_scheme.images:
if image.target_device.name:
LOG.debug("Finally: detaching loop device: %s", image.target_device.name)
try:
bu.deattach_loop(image.target_device.name)
except errors.ProcessExecutionError as e:
LOG.warning(
"Error occured while trying to detach " "loop device %s. Error message: %s",
image.target_device.name,
e,
)
if image.img_tmp_file:
LOG.debug("Finally: removing temporary file: %s", image.img_tmp_file)
try:
os.unlink(image.img_tmp_file)
except OSError:
LOG.debug("Finally: file %s seems does not exist " "or can not be removed", image.img_tmp_file)
LOG.debug("Finally: removing chroot directory: %s", chroot)
try:
os.rmdir(chroot)
except OSError:
LOG.debug("Finally: directory %s seems does not exist " "or can not be removed", chroot)
def do_build_image(self):
"""Building OS images
Includes the following steps
1) create temporary sparse files for all images (truncate)
2) attach temporary files to loop devices (losetup)
3) create file systems on these loop devices
4) create temporary chroot directory
5) mount loop devices into chroot directory
6) install operating system (debootstrap and apt-get)
7) configure OS (clean sources.list and preferences, etc.)
8) umount loop devices
9) resize file systems on loop devices
10) shrink temporary sparse files (images)
11) containerize (gzip) temporary sparse files
12) move temporary gzipped files to their final location
"""
LOG.info('--- Building image (do_build_image) ---')
# TODO(kozhukalov): Implement metadata
# as a pluggable data driver to avoid any fixed format.
metadata = {}
# TODO(kozhukalov): implement this using image metadata
# we need to compare list of packages and repos
LOG.info('*** Checking if image exists ***')
if all([
os.path.exists(img.uri.split('file://', 1)[1])
for img in self.driver.image_scheme.images
]):
LOG.debug('All necessary images are available. '
'Nothing needs to be done.')
return
LOG.debug('At least one of the necessary images is unavailable. '
'Starting build process.')
try:
LOG.debug('Creating temporary chroot directory')
chroot = tempfile.mkdtemp(dir=CONF.image_build_dir,
suffix=CONF.image_build_suffix)
LOG.debug('Temporary chroot: %s', chroot)
proc_path = os.path.join(chroot, 'proc')
LOG.info('*** Preparing image space ***')
for image in self.driver.image_scheme.images:
LOG.debug(
'Creating temporary sparsed file for the '
'image: %s', image.uri)
img_tmp_file = bu.create_sparse_tmp_file(
dir=CONF.image_build_dir, suffix=CONF.image_build_suffix)
LOG.debug('Temporary file: %s', img_tmp_file)
# we need to remember those files
# to be able to shrink them and move in the end
image.img_tmp_file = img_tmp_file
LOG.debug('Looking for a free loop device')
image.target_device.name = bu.get_free_loop_device()
LOG.debug('Attaching temporary image file to free loop device')
bu.attach_file_to_loop(img_tmp_file, str(image.target_device))
# find fs with the same loop device object
# as image.target_device
fs = self.driver.partition_scheme.fs_by_device(
image.target_device)
LOG.debug('Creating file system on the image')
fu.make_fs(fs_type=fs.type,
fs_options=fs.options,
fs_label=fs.label,
dev=str(fs.device))
if fs.type == 'ext4':
LOG.debug('Trying to disable journaling for ext4 '
'in order to speed up the build')
utils.execute('tune2fs', '-O', '^has_journal',
str(fs.device))
# mounting all images into chroot tree
self.mount_target(chroot, treat_mtab=False, pseudo=False)
LOG.info('*** Shipping image content ***')
LOG.debug('Installing operating system into image')
# FIXME(kozhukalov): !!! we need this part to be OS agnostic
# DEBOOTSTRAP
# we use first repo as the main mirror
uri = self.driver.operating_system.repos[0].uri
suite = self.driver.operating_system.repos[0].suite
LOG.debug('Preventing services from being get started')
bu.suppress_services_start(chroot)
LOG.debug('Installing base operating system using debootstrap')
bu.run_debootstrap(uri=uri, suite=suite, chroot=chroot)
# APT-GET
LOG.debug('Configuring apt inside chroot')
LOG.debug('Setting environment variables')
bu.set_apt_get_env()
LOG.debug('Allowing unauthenticated repos')
bu.pre_apt_get(chroot)
for repo in self.driver.operating_system.repos:
LOG.debug('Adding repository source: name={name}, uri={uri},'
'suite={suite}, section={section}'.format(
name=repo.name,
uri=repo.uri,
suite=repo.suite,
section=repo.section))
bu.add_apt_source(name=repo.name,
uri=repo.uri,
suite=repo.suite,
section=repo.section,
chroot=chroot)
LOG.debug('Adding repository preference: '
'name={name}, priority={priority}'.format(
name=repo.name, priority=repo.priority))
if repo.priority is not None:
bu.add_apt_preference(name=repo.name,
priority=repo.priority,
suite=repo.suite,
section=repo.section,
chroot=chroot,
uri=repo.uri)
metadata.setdefault('repos', []).append({
'type': 'deb',
'name': repo.name,
'uri': repo.uri,
'suite': repo.suite,
'section': repo.section,
'priority': repo.priority,
'meta': repo.meta
})
LOG.debug('Preventing services from being get started')
bu.suppress_services_start(chroot)
packages = self.driver.operating_system.packages
metadata['packages'] = packages
# we need /proc to be mounted for apt-get success
utils.makedirs_if_not_exists(proc_path)
fu.mount_bind(chroot, '/proc')
LOG.debug('Installing packages using apt-get: %s',
' '.join(packages))
bu.run_apt_get(chroot, packages=packages)
LOG.debug('Post-install OS configuration')
bu.do_post_inst(chroot)
LOG.debug('Making sure there are no running processes '
'inside chroot before trying to umount chroot')
if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
if not bu.stop_chrooted_processes(chroot,
signal=signal.SIGKILL):
raise errors.UnexpectedProcessError(
'Stopping chrooted processes failed. '
'There are some processes running in chroot %s',
chroot)
LOG.info('*** Finalizing image space ***')
fu.umount_fs(proc_path)
# umounting all loop devices
self.umount_target(chroot, pseudo=False, try_lazy_umount=False)
for image in self.driver.image_scheme.images:
# find fs with the same loop device object
# as image.target_device
fs = self.driver.partition_scheme.fs_by_device(
image.target_device)
if fs.type == 'ext4':
LOG.debug('Trying to re-enable journaling for ext4')
utils.execute('tune2fs', '-O', 'has_journal',
str(fs.device))
LOG.debug('Deattaching loop device from file: %s',
image.img_tmp_file)
bu.deattach_loop(str(image.target_device))
LOG.debug('Shrinking temporary image file: %s',
image.img_tmp_file)
bu.shrink_sparse_file(image.img_tmp_file)
raw_size = os.path.getsize(image.img_tmp_file)
raw_md5 = utils.calculate_md5(image.img_tmp_file, raw_size)
LOG.debug('Containerizing temporary image file: %s',
image.img_tmp_file)
img_tmp_containerized = bu.containerize(
image.img_tmp_file, image.container)
img_containerized = image.uri.split('file://', 1)[1]
# NOTE(kozhukalov): implement abstract publisher
LOG.debug('Moving image file to the final location: %s',
img_containerized)
shutil.move(img_tmp_containerized, img_containerized)
container_size = os.path.getsize(img_containerized)
container_md5 = utils.calculate_md5(img_containerized,
container_size)
metadata.setdefault('images', []).append({
'raw_md5':
raw_md5,
'raw_size':
raw_size,
'raw_name':
None,
'container_name':
os.path.basename(img_containerized),
'container_md5':
container_md5,
'container_size':
container_size,
'container':
image.container,
'format':
image.format
})
# NOTE(kozhukalov): implement abstract publisher
LOG.debug('Image metadata: %s', metadata)
with open(self.driver.metadata_uri.split('file://', 1)[1],
'w') as f:
yaml.safe_dump(metadata, stream=f)
LOG.info('--- Building image END (do_build_image) ---')
except Exception as exc:
LOG.error('Failed to build image: %s', exc)
raise
finally:
LOG.debug('Finally: stopping processes inside chroot: %s', chroot)
if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
bu.stop_chrooted_processes(chroot, signal=signal.SIGKILL)
LOG.debug('Finally: umounting procfs %s', proc_path)
fu.umount_fs(proc_path)
LOG.debug('Finally: umounting chroot tree %s', chroot)
self.umount_target(chroot, pseudo=False, try_lazy_umount=False)
for image in self.driver.image_scheme.images:
LOG.debug('Finally: detaching loop device: %s',
str(image.target_device))
try:
bu.deattach_loop(str(image.target_device))
except errors.ProcessExecutionError as e:
LOG.warning(
'Error occured while trying to detach '
'loop device %s. Error message: %s',
str(image.target_device), e)
LOG.debug('Finally: removing temporary file: %s',
image.img_tmp_file)
try:
os.unlink(image.img_tmp_file)
except OSError:
LOG.debug(
'Finally: file %s seems does not exist '
'or can not be removed', image.img_tmp_file)
LOG.debug('Finally: removing chroot directory: %s', chroot)
try:
os.rmdir(chroot)
except OSError:
LOG.debug(
'Finally: directory %s seems does not exist '
'or can not be removed', chroot)
def do_mkbootstrap(self):
"""Building bootstrap image
Currently supports only Ubuntu-Trusty
Includes the following steps
1) Allocate and configure debootstrap.
2) Install packages
3) Run user-post script(is defined)
4) populate squashfs\init\vmlinuz files
5) create metadata.yaml and pack thats all into tar.gz
"""
LOG.info('--- Building bootstrap image (do_mkbootstrap) ---')
driver_os = self.driver.operating_system
# c_dir = output container directory, where all builded files will
# be stored, before packaging into archive
LOG.debug('Creating bootstrap container folder')
c_dir = bu.mkdtemp_smart(CONF.image_build_dir,
CONF.image_build_suffix + '_container')
try:
chroot = bu.mkdtemp_smart(
CONF.image_build_dir, CONF.image_build_suffix)
self.install_base_os(chroot)
bs_scheme = self.driver.bootstrap_scheme
# init modules, needed for bootstrap. Currently
# we support only one scheme initrd + rootfs + kernel
initrd = filter(lambda x: x.name == 'initrd',
bs_scheme.modules)[0]
rootfs = filter(lambda x: x.name == 'rootfs',
bs_scheme.modules)[0]
metadata = {}
metadata['os'] = driver_os.to_dict()
packages = driver_os.packages
metadata['packages'] = packages
self._set_apt_repos(
chroot, driver_os.repos,
proxies=driver_os.proxies.proxies,
direct_repo_addrs=driver_os.proxies.direct_repo_addr_list)
self._update_metadata_with_repos(
metadata, driver_os.repos)
LOG.debug('Installing packages using apt-get: %s',
' '.join(packages))
# disable hosts/resolv files
bu.propagate_host_resolv_conf(chroot)
bu.run_apt_get(chroot, packages=packages,
attempts=CONF.fetch_packages_attempts)
LOG.debug('Post-install OS configuration')
if hasattr(bs_scheme, 'extra_files') and bs_scheme.extra_files:
for extra in bs_scheme.extra_files:
bu.rsync_inject(extra, chroot)
if (hasattr(bs_scheme, 'root_ssh_authorized_file') and
bs_scheme.root_ssh_authorized_file):
LOG.debug('Put ssh auth file %s',
bs_scheme.root_ssh_authorized_file)
auth_file = os.path.join(chroot, 'root/.ssh/authorized_keys')
utils.makedirs_if_not_exists(os.path.dirname(
auth_file), mode=0o700)
shutil.copy(
bs_scheme.root_ssh_authorized_file,
auth_file)
os.chmod(auth_file, 0o700)
# Allow user to drop and run script inside chroot:
if (hasattr(bs_scheme, 'post_script_file') and
bs_scheme.post_script_file):
bu.run_script_in_chroot(
chroot, bs_scheme.post_script_file)
# Save runtime_uuid into bootstrap
bu.dump_runtime_uuid(bs_scheme.uuid,
os.path.join(chroot,
'etc/nailgun-agent/config.yaml'))
bu.do_post_inst(chroot,
allow_unsigned_file=CONF.allow_unsigned_file,
force_ipv4_file=CONF.force_ipv4_file)
# restore disabled hosts/resolv files
bu.restore_resolv_conf(chroot)
metadata['all_packages'] = bu.get_installed_packages(chroot)
# We need to recompress initramfs with new compression:
bu.recompress_initramfs(
chroot,
compress=initrd.compress_format)
# Bootstrap nodes load the kernel and initramfs via the network,
# therefore remove the kernel and initramfs located in root
# filesystem to make the image smaller (and save the network
# bandwidth and the boot time)
bu.copy_kernel_initramfs(chroot, c_dir, clean=True)
LOG.debug('Making sure there are no running processes '
'inside chroot before trying to umount chroot')
if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
if not bu.stop_chrooted_processes(
chroot, signal=signal.SIGKILL):
raise errors.UnexpectedProcessError(
'Stopping chrooted processes failed. '
'There are some processes running in chroot %s',
chroot)
bu.run_mksquashfs(
chroot, os.path.join(c_dir, os.path.basename(rootfs.uri)),
rootfs.compress_format)
self.dump_mkbootstrap_meta(metadata, c_dir, bs_scheme)
output = bu.save_bs_container(self.driver.output, c_dir,
bs_scheme.container.format)
LOG.info('--- Building bootstrap image END (do_mkbootstrap) ---')
return output
except Exception as exc:
LOG.error('Failed to build bootstrap image: %s', exc)
raise
finally:
LOG.info('Cleanup chroot')
self.destroy_chroot(chroot)
try:
shutil.rmtree(c_dir)
except OSError:
LOG.debug('Finally: directory %s seems does not exist '
'or can not be removed', c_dir)
def run_mksquashfs(chroot, output_name=None, compression_algorithm='xz'):
"""Pack the target system as squashfs using mksquashfs
:param chroot: chroot system, to be squashfs'd
:param output_name: output file name, might be a relative
or an absolute path
The kernel squashfs driver has to match with the user space squasfs tools.
Use the mksquashfs provided by the target distro to achieve this.
(typically the distro maintainers are smart enough to ship the correct
version of mksquashfs)
Use mksquashfs installed in the target system
1)Mount tmpfs under chroot/mnt
2)run mksquashfs inside a chroot
3)move result files to dstdir
"""
if not output_name:
output_name = 'root.squashfs' + six.text_type(uuid.uuid4())
utils.makedirs_if_not_exists(os.path.dirname(output_name))
dstdir = os.path.dirname(output_name)
temp = '.mksquashfs.tmp.' + six.text_type(uuid.uuid4())
s_dst = os.path.join(chroot, 'mnt/dst')
s_src = os.path.join(chroot, 'mnt/src')
try:
fu.mount_fs('tmpfs', 'mnt_{0}'.format(temp),
(os.path.join(chroot, 'mnt')),
'rw,nodev,nosuid,noatime,mode=0755,size=4M')
utils.makedirs_if_not_exists(s_src)
utils.makedirs_if_not_exists(s_dst)
# Bind mount the chroot to avoid including various temporary/virtual
# files (/proc, /sys, /dev, and so on) into the image
fu.mount_fs(None, chroot, s_src, opts='bind')
fu.mount_fs(None, None, s_src, 'remount,bind,ro')
fu.mount_fs(None, dstdir, s_dst, opts='bind')
# run mksquashfs
chroot_squash = os.path.join('/mnt/dst/' + temp)
long_squash = os.path.join(chroot, 'mnt/dst/{0}'.format(temp))
LOG.info('Building squashfs')
utils.execute('chroot',
chroot,
'mksquashfs',
'/mnt/src',
chroot_squash,
'-comp',
compression_algorithm,
'-no-progress',
'-noappend',
logged=True)
# move to result name
LOG.debug('Moving file: %s to: %s', long_squash, output_name)
shutil.move(long_squash, output_name)
except Exception as exc:
LOG.error('squashfs_image build failed: %s', exc)
raise
finally:
LOG.info('squashfs_image clean-up')
stop_chrooted_processes(chroot, signal=signal.SIGTERM)
fu.umount_fs(os.path.join(chroot, 'mnt/dst'))
fu.umount_fs(os.path.join(chroot, 'mnt/src'))
fu.umount_fs(os.path.join(chroot, 'mnt'))
def do_bootloader(self):
LOG.debug('--- Installing bootloader (do_bootloader) ---')
chroot = '/tmp/target'
self.mount_target(chroot)
mount2uuid = {}
for fs in self.driver.partition_scheme.fss:
mount2uuid[fs.mount] = utils.execute(
'blkid', '-o', 'value', '-s', 'UUID', fs.device,
check_exit_code=[0])[0].strip()
if '/' not in mount2uuid:
raise errors.WrongPartitionSchemeError(
'Error: device with / mountpoint has not been found')
grub = self.driver.grub
guessed_version = gu.guess_grub_version(chroot=chroot)
if guessed_version != grub.version:
grub.version = guessed_version
LOG.warning('Grub version differs from which the operating system '
'should have by default. Found version in image: '
'{0}'.format(guessed_version))
boot_device = self.driver.partition_scheme.boot_device(grub.version)
install_devices = [d.name for d in self.driver.partition_scheme.parteds
if d.install_bootloader]
grub.append_kernel_params('root=UUID=%s ' % mount2uuid['/'])
kernel = grub.kernel_name or \
gu.guess_kernel(chroot=chroot, regexp=grub.kernel_regexp)
initrd = grub.initrd_name or \
gu.guess_initrd(chroot=chroot, regexp=grub.initrd_regexp)
if grub.version == 1:
gu.grub1_cfg(kernel=kernel, initrd=initrd,
kernel_params=grub.kernel_params, chroot=chroot,
grub_timeout=CONF.grub_timeout)
gu.grub1_install(install_devices, boot_device, chroot=chroot)
else:
# TODO(kozhukalov): implement which kernel to use by default
# Currently only grub1_cfg accepts kernel and initrd parameters.
gu.grub2_cfg(kernel_params=grub.kernel_params, chroot=chroot,
grub_timeout=CONF.grub_timeout)
gu.grub2_install(install_devices, chroot=chroot)
if CONF.fix_udev_net_rules:
# FIXME(agordeev) There's no convenient way to perfrom NIC
# remapping in Ubuntu, so injecting files prior the first boot
# should work
with open(chroot + '/etc/udev/rules.d/70-persistent-net.rules',
'wt', encoding='utf-8') as f:
f.write(u'# Generated by fuel-agent during provisioning: '
u'BEGIN\n')
# pattern is aa:bb:cc:dd:ee:ff_eth0,aa:bb:cc:dd:ee:ff_eth1
for mapping in self.driver.configdrive_scheme.\
common.udevrules.split(','):
mac_addr, nic_name = mapping.split('_')
f.write(u'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", '
u'ATTR{address}=="%s", ATTR{type}=="1", '
u'KERNEL=="eth*", NAME="%s"\n' % (mac_addr,
nic_name))
f.write(
u'# Generated by fuel-agent during provisioning: END\n')
# FIXME(agordeev): Disable net-generator that will add new etries
# to 70-persistent-net.rules
with open(chroot +
'/etc/udev/rules.d/75-persistent-net-generator.rules',
'wt', encoding='utf-8') as f:
f.write(u'# Generated by fuel-agent during provisioning:\n'
u'# DO NOT DELETE. It is needed to disable '
u'net-generator\n')
# FIXME(kozhukalov): Prevent nailgun-agent from doing anything.
# This ugly hack is to be used together with the command removing
# this lock file not earlier than /etc/rc.local
# The reason for this hack to appear is to prevent nailgun-agent from
# changing mcollective config at the same time when cloud-init
# does the same. Otherwise, we can end up with corrupted mcollective
# config. For details see https://bugs.launchpad.net/fuel/+bug/1449186
LOG.debug('Preventing nailgun-agent from doing '
'anything until it is unlocked')
utils.makedirs_if_not_exists(os.path.join(chroot, 'etc/nailgun-agent'))
with open(os.path.join(chroot, 'etc/nailgun-agent/nodiscover'), 'w'):
pass
# FIXME(kozhukalov): When we have just os-root fs image and don't have
# os-var-log fs image while / and /var/log are supposed to be
# separate file systems and os-var-log is mounted into
# non-empty directory on the / file system, those files in /var/log
# directory become unavailable.
# The thing is that among those file there is /var/log/upstart
# where upstart daemon writes its logs. We have specific upstart job
# which is to flush open files once all file systems are mounted.
# This job needs upstart directory to be available on os-var-log
# file system.
# This is just a temporary fix and correct fix will be available soon
# via updates.
utils.execute('mkdir', '-p', chroot + '/var/log/upstart')
with open(chroot + '/etc/fstab', 'wt', encoding='utf-8') as f:
for fs in self.driver.partition_scheme.fss:
# TODO(kozhukalov): Think of improving the logic so as to
# insert a meaningful fsck order value which is last zero
# at fstab line. Currently we set it into 0 which means
# a corresponding file system will never be checked. We assume
# puppet or other configuration tool will care of it.
if fs.mount == '/':
f.write('UUID=%s %s %s defaults,errors=panic 0 0\n' %
(mount2uuid[fs.mount], fs.mount, fs.type))
else:
f.write('UUID=%s %s %s defaults 0 0\n' %
(mount2uuid[fs.mount], fs.mount, fs.type))
self.umount_target(chroot)
def do_build_image(self):
"""Building OS images
Includes the following steps
1) create temporary sparse files for all images (truncate)
2) attach temporary files to loop devices (losetup)
3) create file systems on these loop devices
4) create temporary chroot directory
5) mount loop devices into chroot directory
6) install operating system (debootstrap and apt-get)
7) configure OS (clean sources.list and preferences, etc.)
8) umount loop devices
9) resize file systems on loop devices
10) shrink temporary sparse files (images)
11) containerize (gzip) temporary sparse files
12) move temporary gzipped files to their final location
"""
LOG.info('--- Building image (do_build_image) ---')
# TODO(kozhukalov): Implement metadata
# as a pluggable data driver to avoid any fixed format.
metadata = {}
metadata['os'] = self.driver.operating_system.to_dict()
# TODO(kozhukalov): implement this using image metadata
# we need to compare list of packages and repos
LOG.info('*** Checking if image exists ***')
if all([os.path.exists(img.uri.split('file://', 1)[1])
for img in self.driver.image_scheme.images]):
LOG.debug('All necessary images are available. '
'Nothing needs to be done.')
return
LOG.debug('At least one of the necessary images is unavailable. '
'Starting build process.')
try:
LOG.debug('Creating temporary chroot directory')
utils.makedirs_if_not_exists(CONF.image_build_dir)
chroot = tempfile.mkdtemp(
dir=CONF.image_build_dir, suffix=CONF.image_build_suffix)
LOG.debug('Temporary chroot: %s', chroot)
proc_path = os.path.join(chroot, 'proc')
LOG.info('*** Preparing image space ***')
for image in self.driver.image_scheme.images:
LOG.debug('Creating temporary sparsed file for the '
'image: %s', image.uri)
img_tmp_file = bu.create_sparse_tmp_file(
dir=CONF.image_build_dir, suffix=CONF.image_build_suffix,
size=CONF.sparse_file_size)
LOG.debug('Temporary file: %s', img_tmp_file)
# we need to remember those files
# to be able to shrink them and move in the end
image.img_tmp_file = img_tmp_file
LOG.debug('Looking for a free loop device')
image.target_device.name = bu.get_free_loop_device(
loop_device_major_number=CONF.loop_device_major_number,
max_loop_devices_count=CONF.max_loop_devices_count)
LOG.debug('Attaching temporary image file to free loop device')
bu.attach_file_to_loop(img_tmp_file, str(image.target_device))
# find fs with the same loop device object
# as image.target_device
fs = self.driver.partition_scheme.fs_by_device(
image.target_device)
LOG.debug('Creating file system on the image')
fu.make_fs(
fs_type=fs.type,
fs_options=fs.options,
fs_label=fs.label,
dev=str(fs.device))
if fs.type == 'ext4':
LOG.debug('Trying to disable journaling for ext4 '
'in order to speed up the build')
utils.execute('tune2fs', '-O', '^has_journal',
str(fs.device))
# mounting all images into chroot tree
self.mount_target(chroot, treat_mtab=False, pseudo=False)
LOG.info('*** Shipping image content ***')
LOG.debug('Installing operating system into image')
# FIXME(kozhukalov): !!! we need this part to be OS agnostic
# DEBOOTSTRAP
# we use first repo as the main mirror
uri = self.driver.operating_system.repos[0].uri
suite = self.driver.operating_system.repos[0].suite
LOG.debug('Preventing services from being get started')
bu.suppress_services_start(chroot)
LOG.debug('Installing base operating system using debootstrap')
bu.run_debootstrap(uri=uri, suite=suite, chroot=chroot,
attempts=CONF.fetch_packages_attempts)
# APT-GET
LOG.debug('Configuring apt inside chroot')
LOG.debug('Setting environment variables')
bu.set_apt_get_env()
LOG.debug('Allowing unauthenticated repos')
bu.pre_apt_get(chroot,
allow_unsigned_file=CONF.allow_unsigned_file,
force_ipv4_file=CONF.force_ipv4_file)
for repo in self.driver.operating_system.repos:
LOG.debug('Adding repository source: name={name}, uri={uri},'
'suite={suite}, section={section}'.format(
name=repo.name, uri=repo.uri,
suite=repo.suite, section=repo.section))
bu.add_apt_source(
name=repo.name,
uri=repo.uri,
suite=repo.suite,
section=repo.section,
chroot=chroot)
LOG.debug('Adding repository preference: '
'name={name}, priority={priority}'.format(
name=repo.name, priority=repo.priority))
if repo.priority is not None:
bu.add_apt_preference(
name=repo.name,
priority=repo.priority,
suite=repo.suite,
section=repo.section,
chroot=chroot,
uri=repo.uri)
metadata.setdefault('repos', []).append({
'type': 'deb',
'name': repo.name,
'uri': repo.uri,
'suite': repo.suite,
'section': repo.section,
'priority': repo.priority,
'meta': repo.meta})
LOG.debug('Preventing services from being get started')
bu.suppress_services_start(chroot)
packages = self.driver.operating_system.packages
metadata['packages'] = packages
# we need /proc to be mounted for apt-get success
utils.makedirs_if_not_exists(proc_path)
fu.mount_bind(chroot, '/proc')
bu.populate_basic_dev(chroot)
LOG.debug('Installing packages using apt-get: %s',
' '.join(packages))
bu.run_apt_get(chroot, packages=packages,
attempts=CONF.fetch_packages_attempts)
LOG.debug('Post-install OS configuration')
bu.do_post_inst(chroot,
allow_unsigned_file=CONF.allow_unsigned_file,
force_ipv4_file=CONF.force_ipv4_file)
LOG.debug('Making sure there are no running processes '
'inside chroot before trying to umount chroot')
if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
if not bu.stop_chrooted_processes(
chroot, signal=signal.SIGKILL):
raise errors.UnexpectedProcessError(
'Stopping chrooted processes failed. '
'There are some processes running in chroot %s',
chroot)
LOG.info('*** Finalizing image space ***')
fu.umount_fs(proc_path)
# umounting all loop devices
self.umount_target(chroot, pseudo=False)
for image in self.driver.image_scheme.images:
# find fs with the same loop device object
# as image.target_device
fs = self.driver.partition_scheme.fs_by_device(
image.target_device)
if fs.type == 'ext4':
LOG.debug('Trying to re-enable journaling for ext4')
utils.execute('tune2fs', '-O', 'has_journal',
str(fs.device))
LOG.debug('Deattaching loop device from file: %s',
image.img_tmp_file)
bu.deattach_loop(str(image.target_device))
LOG.debug('Shrinking temporary image file: %s',
image.img_tmp_file)
bu.shrink_sparse_file(image.img_tmp_file)
raw_size = os.path.getsize(image.img_tmp_file)
raw_md5 = utils.calculate_md5(image.img_tmp_file, raw_size)
LOG.debug('Containerizing temporary image file: %s',
image.img_tmp_file)
img_tmp_containerized = bu.containerize(
image.img_tmp_file, image.container,
chunk_size=CONF.data_chunk_size)
img_containerized = image.uri.split('file://', 1)[1]
# NOTE(kozhukalov): implement abstract publisher
LOG.debug('Moving image file to the final location: %s',
img_containerized)
shutil.move(img_tmp_containerized, img_containerized)
container_size = os.path.getsize(img_containerized)
container_md5 = utils.calculate_md5(
img_containerized, container_size)
metadata.setdefault('images', []).append({
'raw_md5': raw_md5,
'raw_size': raw_size,
'raw_name': None,
'container_name': os.path.basename(img_containerized),
'container_md5': container_md5,
'container_size': container_size,
'container': image.container,
'format': image.format})
# NOTE(kozhukalov): implement abstract publisher
LOG.debug('Image metadata: %s', metadata)
with open(self.driver.metadata_uri.split('file://', 1)[1],
'wt', encoding='utf-8') as f:
yaml.safe_dump(metadata, stream=f)
LOG.info('--- Building image END (do_build_image) ---')
except Exception as exc:
LOG.error('Failed to build image: %s', exc)
raise
finally:
LOG.debug('Finally: stopping processes inside chroot: %s', chroot)
if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
bu.stop_chrooted_processes(chroot, signal=signal.SIGKILL)
LOG.debug('Finally: umounting procfs %s', proc_path)
fu.umount_fs(proc_path)
LOG.debug('Finally: umounting chroot tree %s', chroot)
self.umount_target(chroot, pseudo=False)
for image in self.driver.image_scheme.images:
LOG.debug('Finally: detaching loop device: %s',
str(image.target_device))
try:
bu.deattach_loop(str(image.target_device))
except errors.ProcessExecutionError as e:
LOG.warning('Error occured while trying to detach '
'loop device %s. Error message: %s',
str(image.target_device), e)
LOG.debug('Finally: removing temporary file: %s',
image.img_tmp_file)
try:
os.unlink(image.img_tmp_file)
except OSError:
LOG.debug('Finally: file %s seems does not exist '
'or can not be removed', image.img_tmp_file)
LOG.debug('Finally: removing chroot directory: %s', chroot)
try:
os.rmdir(chroot)
except OSError:
LOG.debug('Finally: directory %s seems does not exist '
'or can not be removed', chroot)
def test_makedirs_if_not_exists_mode_given(
self, mock_isdir, mock_makedirs):
utils.makedirs_if_not_exists('/fake/path', mode=0o000)
mock_isdir.assert_called_once_with('/fake/path')
mock_makedirs.assert_called_once_with('/fake/path', mode=0o000)
def install_base_os(self, chroot):
"""Bootstrap a basic Linux system
:param chroot directory where the installed OS can be found
For now only Ubuntu is supported.
Note: the data gets written to a different location (a set of
ext4 images located in the image_build_dir directory)
Includes the following steps
1) create temporary sparse files for all images (truncate)
2) attach temporary files to loop devices (losetup)
3) create file systems on these loop devices
4) create temporary chroot directory
5) mount loop devices into chroot directory
6) install operating system (debootstrap and apt-get)
"""
LOG.info('*** Preparing image space ***')
for image in self.driver.image_scheme.images:
LOG.debug('Creating temporary sparsed file for the '
'image: %s', image.uri)
img_tmp_file = bu.create_sparse_tmp_file(
dir=CONF.image_build_dir, suffix=CONF.image_build_suffix,
size=CONF.sparse_file_size)
LOG.debug('Temporary file: %s', img_tmp_file)
# we need to remember those files
# to be able to shrink them and move in the end
image.img_tmp_file = img_tmp_file
image.target_device.name = \
bu.attach_file_to_free_loop_device(
img_tmp_file,
max_loop_devices_count=CONF.max_loop_devices_count,
loop_device_major_number=CONF.loop_device_major_number,
max_attempts=CONF.max_allowed_attempts_attach_image)
# find fs with the same loop device object
# as image.target_device
fs = self.driver.partition_scheme.fs_by_device(
image.target_device)
LOG.debug('Creating file system on the image')
fu.make_fs(
fs_type=fs.type,
fs_options=fs.options,
fs_label=fs.label,
dev=six.text_type(fs.device))
if fs.type == 'ext4':
LOG.debug('Trying to disable journaling for ext4 '
'in order to speed up the build')
utils.execute('tune2fs', '-O', '^has_journal',
six.text_type(fs.device))
# mounting all images into chroot tree
self.mount_target(chroot, treat_mtab=False, pseudo=False)
LOG.info('Installing BASE operating system into image')
# FIXME(kozhukalov): !!! we need this part to be OS agnostic
# DEBOOTSTRAP
# we use first repo as the main mirror
uri = self.driver.operating_system.repos[0].uri
suite = self.driver.operating_system.repos[0].suite
proxies = self.driver.operating_system.proxies
LOG.debug('Preventing services from being get started')
bu.suppress_services_start(chroot)
LOG.debug('Installing base operating system using debootstrap')
bu.run_debootstrap(uri=uri, suite=suite, chroot=chroot,
attempts=CONF.fetch_packages_attempts,
proxies=proxies.proxies,
direct_repo_addr=proxies.direct_repo_addr_list)
# APT-GET
LOG.debug('Configuring apt inside chroot')
LOG.debug('Setting environment variables')
bu.set_apt_get_env()
LOG.debug('Allowing unauthenticated repos')
bu.pre_apt_get(chroot,
allow_unsigned_file=CONF.allow_unsigned_file,
force_ipv4_file=CONF.force_ipv4_file,
proxies=proxies.proxies,
direct_repo_addr=proxies.direct_repo_addr_list)
# we need /proc to be mounted for apt-get success
LOG.debug('Preventing services from being get started')
bu.suppress_services_start(chroot)
utils.makedirs_if_not_exists(os.path.join(chroot, 'proc'))
# we need /proc to be mounted for apt-get success
fu.mount_bind(chroot, '/proc')
bu.populate_basic_dev(chroot)
def do_mkbootstrap(self):
"""Building bootstrap image
Currently supports only Ubuntu-Trusty
Includes the following steps
1) Allocate and configure debootstrap.
2) Install packages
3) Run user-post script(is defined)
4) populate squashfs\init\vmlinuz files
5) create metadata.yaml and pack thats all into tar.gz
"""
LOG.info('--- Building bootstrap image (do_mkbootstrap) ---')
driver_os = self.driver.operating_system
# c_dir = output container directory, where all builded files will
# be stored, before packaging into archive
LOG.debug('Creating bootstrap container folder')
c_dir = bu.mkdtemp_smart(CONF.image_build_dir,
CONF.image_build_suffix + '_container')
try:
chroot = bu.mkdtemp_smart(
CONF.image_build_dir, CONF.image_build_suffix)
self.install_base_os(chroot)
bs_scheme = self.driver.bootstrap_scheme
# init modules, needed for bootstrap. Currently
# we support only one scheme initrd + rootfs + kernel
initrd = filter(lambda x: x.name == 'initrd',
bs_scheme.modules)[0]
rootfs = filter(lambda x: x.name == 'rootfs',
bs_scheme.modules)[0]
metadata = {}
metadata['os'] = driver_os.to_dict()
packages = driver_os.packages
metadata['packages'] = packages
self._set_apt_repos(
chroot, driver_os.repos,
proxies=driver_os.proxies.proxies,
direct_repo_addrs=driver_os.proxies.direct_repo_addr_list)
self._update_metadata_with_repos(
metadata, driver_os.repos)
LOG.debug('Installing packages using apt-get: %s',
' '.join(packages))
# disable hosts/resolv files
bu.propagate_host_resolv_conf(chroot)
if hasattr(bs_scheme, 'certs') and bs_scheme.certs:
bu.copy_update_certs(bs_scheme.certs, chroot)
bu.run_apt_get(chroot, packages=packages,
attempts=CONF.fetch_packages_attempts)
LOG.debug('Post-install OS configuration')
if hasattr(bs_scheme, 'extra_files') and bs_scheme.extra_files:
for extra in bs_scheme.extra_files:
bu.rsync_inject(extra, chroot)
if (hasattr(bs_scheme, 'root_ssh_authorized_file') and
bs_scheme.root_ssh_authorized_file):
LOG.debug('Put ssh auth file %s',
bs_scheme.root_ssh_authorized_file)
auth_file = os.path.join(chroot, 'root/.ssh/authorized_keys')
utils.makedirs_if_not_exists(os.path.dirname(
auth_file), mode=0o700)
shutil.copy(
bs_scheme.root_ssh_authorized_file,
auth_file)
os.chmod(auth_file, 0o700)
# Allow user to drop and run script inside chroot:
if (hasattr(bs_scheme, 'post_script_file') and
bs_scheme.post_script_file):
bu.run_script_in_chroot(
chroot, bs_scheme.post_script_file)
# Save runtime_uuid into bootstrap
bu.dump_runtime_uuid(bs_scheme.uuid,
os.path.join(chroot,
'etc/nailgun-agent/config.yaml'))
# NOTE(sslypushenko) Preferred names in LVM config should updated
# due to point LVM to work only with /dev/mapper folder
bu.override_lvm_config(
chroot,
{'devices': {
'preferred_names': CONF.mpath_lvm_preferred_names}},
lvm_conf_path=CONF.lvm_conf_path)
root = driver_os.get_user_by_name('root')
bu.do_post_inst(chroot,
hashed_root_password=root.hashed_password,
allow_unsigned_file=CONF.allow_unsigned_file,
force_ipv4_file=CONF.force_ipv4_file,
add_multipath_conf=False)
# restore disabled hosts/resolv files
bu.restore_resolv_conf(chroot)
metadata['all_packages'] = bu.get_installed_packages(chroot)
# We need to recompress initramfs with new compression:
bu.recompress_initramfs(
chroot,
compress=initrd.compress_format)
# Bootstrap nodes load the kernel and initramfs via the network,
# therefore remove the kernel and initramfs located in root
# filesystem to make the image smaller (and save the network
# bandwidth and the boot time)
bu.copy_kernel_initramfs(chroot, c_dir, clean=True)
LOG.debug('Making sure there are no running processes '
'inside chroot before trying to umount chroot')
if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
if not bu.stop_chrooted_processes(
chroot, signal=signal.SIGKILL):
raise errors.UnexpectedProcessError(
'Stopping chrooted processes failed. '
'There are some processes running in chroot %s',
chroot)
bu.run_mksquashfs(
chroot, os.path.join(c_dir, os.path.basename(rootfs.uri)),
rootfs.compress_format)
self.dump_mkbootstrap_meta(metadata, c_dir, bs_scheme)
output = bu.save_bs_container(self.driver.output, c_dir,
bs_scheme.container.format)
LOG.info('--- Building bootstrap image END (do_mkbootstrap) ---')
return output
except Exception as exc:
LOG.error('Failed to build bootstrap image: %s', exc)
raise
finally:
LOG.info('Cleanup chroot')
self.destroy_chroot(chroot)
try:
shutil.rmtree(c_dir)
except OSError:
LOG.debug('Finally: directory %s seems does not exist '
'or can not be removed', c_dir)
def do_bootloader(self):
LOG.debug('--- Installing bootloader (do_bootloader) ---')
chroot = '/tmp/target'
self.mount_target(chroot)
mount2uuid = {}
for fs in self.driver.partition_scheme.fss:
mount2uuid[fs.mount] = utils.execute(
'blkid', '-c', '/dev/null', '-o', 'value',
'-s', 'UUID', fs.device,
check_exit_code=[0])[0].strip()
if '/' not in mount2uuid:
raise errors.WrongPartitionSchemeError(
'Error: device with / mountpoint has not been found')
# NOTE(sslypushenko) Due to possible races between LVM and multipath,
# we need to adjust LVM devices filter.
# This code is required only for Ubuntu 14.04, because in trusty,
# LVM filters, does not recognize partions on multipath devices
# out of the box. It is fixed in latest LVM versions
multipath_devs = [parted.name
for parted in self.driver.partition_scheme.parteds
if hw.is_multipath_device(parted.name)]
# If there are no multipath devices on the node, we should not do
# anything to prevent regression.
if multipath_devs:
# We need to explicitly whitelist each non-mutlipath device
lvm_filter = []
for parted in self.driver.partition_scheme.parteds:
device = parted.name
if device in multipath_devs:
continue
# We use devlinks from /dev/disk/by-id instead of /dev/sd*,
# because the first one are persistent.
devlinks_by_id = [
link for link in hw.udevreport(device).get('DEVLINKS', [])
if link.startswith('/dev/disk/by-id/')]
for link in devlinks_by_id:
lvm_filter.append(
'a|^{}(p)?(-part)?[0-9]*|'.format(link))
# Multipath devices should be whitelisted. All other devlinks
# should be blacklisted, to prevent LVM from grubbing underlying
# multipath devices.
lvm_filter.extend(CONF.lvm_filter_for_mpath)
# Setting devices/preferred_names also helps LVM to find devices by
# the proper devlinks
bu.override_lvm_config(
chroot,
{'devices': {
'scan': CONF.mpath_lvm_scan_dirs,
'global_filter': lvm_filter,
'preferred_names': CONF.mpath_lvm_preferred_names}},
lvm_conf_path=CONF.lvm_conf_path,
update_initramfs=True)
grub = self.driver.grub
guessed_version = gu.guess_grub_version(chroot=chroot)
if guessed_version != grub.version:
grub.version = guessed_version
LOG.warning('Grub version differs from which the operating system '
'should have by default. Found version in image: '
'{0}'.format(guessed_version))
boot_device = self.driver.partition_scheme.boot_device(grub.version)
install_devices = [d.name for d in self.driver.partition_scheme.parteds
if d.install_bootloader]
grub.append_kernel_params('root=UUID=%s ' % mount2uuid['/'])
kernel = grub.kernel_name or gu.guess_kernel(chroot=chroot,
regexp=grub.kernel_regexp)
initrd = grub.initrd_name or gu.guess_initrd(chroot=chroot,
regexp=grub.initrd_regexp)
if grub.version == 1:
gu.grub1_cfg(kernel=kernel, initrd=initrd,
kernel_params=grub.kernel_params, chroot=chroot,
grub_timeout=CONF.grub_timeout)
gu.grub1_install(install_devices, boot_device, chroot=chroot)
else:
# TODO(kozhukalov): implement which kernel to use by default
# Currently only grub1_cfg accepts kernel and initrd parameters.
gu.grub2_cfg(kernel_params=grub.kernel_params, chroot=chroot,
grub_timeout=CONF.grub_timeout)
gu.grub2_install(install_devices, chroot=chroot)
if CONF.fix_udev_net_rules:
# FIXME(agordeev) There's no convenient way to perfrom NIC
# remapping in Ubuntu, so injecting files prior the first boot
# should work
with open(chroot + '/etc/udev/rules.d/70-persistent-net.rules',
'wt', encoding='utf-8') as f:
f.write(u'# Generated by fuel-agent during provisioning: '
u'BEGIN\n')
# pattern is aa:bb:cc:dd:ee:ff_eth0,aa:bb:cc:dd:ee:ff_eth1
for mapping in self.driver.configdrive_scheme. \
common.udevrules.split(','):
mac_addr, nic_name = mapping.split('_')
f.write(u'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", '
u'ATTR{address}=="%s", ATTR{type}=="1", '
u'KERNEL=="eth*", NAME="%s"\n' % (mac_addr,
nic_name))
f.write(
u'# Generated by fuel-agent during provisioning: END\n')
# FIXME(agordeev): Disable net-generator that will add new etries
# to 70-persistent-net.rules
with open(chroot + '/etc/udev/rules.d/'
'75-persistent-net-generator.rules', 'wt',
encoding='utf-8') as f:
f.write(u'# Generated by fuel-agent during provisioning:\n'
u'# DO NOT DELETE. It is needed to disable '
u'net-generator\n')
# FIXME(kozhukalov): Prevent nailgun-agent from doing anything.
# This ugly hack is to be used together with the command removing
# this lock file not earlier than /etc/rc.local
# The reason for this hack to appear is to prevent nailgun-agent from
# changing mcollective config at the same time when cloud-init
# does the same. Otherwise, we can end up with corrupted mcollective
# config. For details see https://bugs.launchpad.net/fuel/+bug/1449186
LOG.debug('Preventing nailgun-agent from doing '
'anything until it is unlocked')
utils.makedirs_if_not_exists(os.path.join(chroot, 'etc/nailgun-agent'))
with open(os.path.join(chroot, 'etc/nailgun-agent/nodiscover'), 'w'):
pass
# FIXME(kozhukalov): When we have just os-root fs image and don't have
# os-var-log fs image while / and /var/log are supposed to be
# separate file systems and os-var-log is mounted into
# non-empty directory on the / file system, those files in /var/log
# directory become unavailable.
# The thing is that among those file there is /var/log/upstart
# where upstart daemon writes its logs. We have specific upstart job
# which is to flush open files once all file systems are mounted.
# This job needs upstart directory to be available on os-var-log
# file system.
# This is just a temporary fix and correct fix will be available soon
# via updates.
utils.execute('mkdir', '-p', chroot + '/var/log/upstart')
with open(chroot + '/etc/fstab', 'wt', encoding='utf-8') as f:
for fs in self.driver.partition_scheme.fss:
# TODO(kozhukalov): Think of improving the logic so as to
# insert a meaningful fsck order value which is last zero
# at fstab line. Currently we set it into 0 which means
# a corresponding file system will never be checked. We assume
# puppet or other configuration tool will care of it.
if fs.mount == '/':
f.write(u'UUID=%s %s %s defaults,errors=panic 0 0\n' %
(mount2uuid[fs.mount], fs.mount, fs.type))
else:
f.write(u'UUID=%s %s %s defaults 0 0\n' %
(mount2uuid[fs.mount], fs.mount, fs.type))
self.umount_target(chroot)
def install_base_os(self, chroot):
"""Bootstrap a basic Linux system
:param chroot directory where the installed OS can be found
For now only Ubuntu is supported.
Note: the data gets written to a different location (a set of
ext4 images located in the image_build_dir directory)
Includes the following steps
1) create temporary sparse files for all images (truncate)
2) attach temporary files to loop devices (losetup)
3) create file systems on these loop devices
4) create temporary chroot directory
5) mount loop devices into chroot directory
6) install operating system (debootstrap and apt-get)
"""
LOG.info('*** Preparing image space ***')
for image in self.driver.image_scheme.images:
LOG.debug('Creating temporary sparsed file for the '
'image: %s', image.uri)
img_tmp_file = bu.create_sparse_tmp_file(
dir=CONF.image_build_dir, suffix=CONF.image_build_suffix,
size=CONF.sparse_file_size)
LOG.debug('Temporary file: %s', img_tmp_file)
# we need to remember those files
# to be able to shrink them and move in the end
image.img_tmp_file = img_tmp_file
image.target_device.name = \
bu.attach_file_to_free_loop_device(
img_tmp_file,
max_loop_devices_count=CONF.max_loop_devices_count,
loop_device_major_number=CONF.loop_device_major_number,
max_attempts=CONF.max_allowed_attempts_attach_image)
# find fs with the same loop device object
# as image.target_device
fs = self.driver.partition_scheme.fs_by_device(
image.target_device)
LOG.debug('Creating file system on the image')
fu.make_fs(
fs_type=fs.type,
fs_options=fs.options,
fs_label=fs.label,
dev=six.text_type(fs.device))
if fs.type == 'ext4':
LOG.debug('Trying to disable journaling for ext4 '
'in order to speed up the build')
utils.execute('tune2fs', '-O', '^has_journal',
six.text_type(fs.device))
# mounting all images into chroot tree
self.mount_target(chroot, treat_mtab=False, pseudo=False)
LOG.info('Installing BASE operating system into image')
# FIXME(kozhukalov): !!! we need this part to be OS agnostic
# DEBOOTSTRAP
# we use first repo as the main mirror
uri = self.driver.operating_system.repos[0].uri
suite = self.driver.operating_system.repos[0].suite
proxies = self.driver.operating_system.proxies
LOG.debug('Preventing services from being get started')
bu.suppress_services_start(chroot)
LOG.debug('Installing base operating system using debootstrap')
bu.run_debootstrap(uri=uri, suite=suite, chroot=chroot,
attempts=CONF.fetch_packages_attempts,
proxies=proxies.proxies,
direct_repo_addr=proxies.direct_repo_addr_list)
# APT-GET
LOG.debug('Configuring apt inside chroot')
LOG.debug('Setting environment variables')
bu.set_apt_get_env()
LOG.debug('Allowing unauthenticated repos')
bu.pre_apt_get(chroot,
allow_unsigned_file=CONF.allow_unsigned_file,
force_ipv4_file=CONF.force_ipv4_file,
proxies=proxies.proxies,
direct_repo_addr=proxies.direct_repo_addr_list)
# we need /proc to be mounted for apt-get success
LOG.debug('Preventing services from being get started')
bu.suppress_services_start(chroot)
utils.makedirs_if_not_exists(os.path.join(chroot, 'proc'))
# we need /proc to be mounted for apt-get success
fu.mount_bind(chroot, '/proc')
bu.populate_basic_dev(chroot)
def do_bootloader(self):
LOG.debug('--- Installing bootloader (do_bootloader) ---')
chroot = '/tmp/target'
self.mount_target(chroot)
mount2uuid = {}
for fs in self.driver.partition_scheme.fss:
mount2uuid[fs.mount] = utils.execute(
'blkid', '-o', 'value', '-s', 'UUID', fs.device,
check_exit_code=[0])[0].strip()
grub = self.driver.grub
grub.version = gu.guess_grub_version(chroot=chroot)
boot_device = self.driver.partition_scheme.boot_device(grub.version)
install_devices = [d.name for d in self.driver.partition_scheme.parteds
if d.install_bootloader]
grub.append_kernel_params('root=UUID=%s ' % mount2uuid['/'])
kernel = grub.kernel_name or \
gu.guess_kernel(chroot=chroot, regexp=grub.kernel_regexp)
initrd = grub.initrd_name or \
gu.guess_initrd(chroot=chroot, regexp=grub.initrd_regexp)
if grub.version == 1:
gu.grub1_cfg(kernel=kernel, initrd=initrd,
kernel_params=grub.kernel_params, chroot=chroot)
gu.grub1_install(install_devices, boot_device, chroot=chroot)
else:
# TODO(kozhukalov): implement which kernel to use by default
# Currently only grub1_cfg accepts kernel and initrd parameters.
gu.grub2_cfg(kernel_params=grub.kernel_params, chroot=chroot)
gu.grub2_install(install_devices, chroot=chroot)
# FIXME(agordeev) There's no convenient way to perfrom NIC remapping in
# Ubuntu, so injecting files prior the first boot should work
with open(chroot + '/etc/udev/rules.d/70-persistent-net.rules',
'w') as f:
f.write('# Generated by fuel-agent during provisioning: BEGIN\n')
# pattern is aa:bb:cc:dd:ee:ff_eth0,aa:bb:cc:dd:ee:ff_eth1
for mapping in self.driver.configdrive_scheme.\
common.udevrules.split(','):
mac_addr, nic_name = mapping.split('_')
f.write('SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", '
'ATTR{address}=="%s", ATTR{type}=="1", KERNEL=="eth*",'
' NAME="%s"\n' % (mac_addr, nic_name))
f.write('# Generated by fuel-agent during provisioning: END\n')
# FIXME(agordeev): Disable net-generator that will add new etries to
# 70-persistent-net.rules
with open(chroot +
'/etc/udev/rules.d/75-persistent-net-generator.rules',
'w') as f:
f.write('# Generated by fuel-agent during provisioning:\n'
'# DO NOT DELETE. It is needed to disable net-generator\n')
# FIXME(kozhukalov): Prevent nailgun-agent from doing anything.
# This ugly hack is to be used together with the command removing
# this lock file not earlier than /etc/rc.local
# The reason for this hack to appear is to prevent nailgun-agent from
# changing mcollective config at the same time when cloud-init
# does the same. Otherwise, we can end up with corrupted mcollective
# config. For details see https://bugs.launchpad.net/fuel/+bug/1449186
LOG.debug('Preventing nailgun-agent from doing '
'anything until it is unlocked')
utils.makedirs_if_not_exists(os.path.join(chroot, 'etc/nailgun-agent'))
with open(os.path.join(chroot, 'etc/nailgun-agent/nodiscover'), 'w'):
pass
with open(chroot + '/etc/fstab', 'wb') as f:
for fs in self.driver.partition_scheme.fss:
# TODO(kozhukalov): Think of improving the logic so as to
# insert a meaningful fsck order value which is last zero
# at fstab line. Currently we set it into 0 which means
# a corresponding file system will never be checked. We assume
# puppet or other configuration tool will care of it.
f.write('UUID=%s %s %s defaults 0 0\n' %
(mount2uuid[fs.mount], fs.mount, fs.type))
self.umount_target(chroot)
def do_bootloader(self):
LOG.debug('--- Installing bootloader (do_bootloader) ---')
chroot = '/tmp/target'
self.mount_target(chroot)
mount2uuid = {}
for fs in self.driver.partition_scheme.fss:
mount2uuid[fs.mount] = utils.execute(
'blkid',
'-o',
'value',
'-s',
'UUID',
fs.device,
check_exit_code=[0])[0].strip()
grub = self.driver.grub
grub.version = gu.guess_grub_version(chroot=chroot)
boot_device = self.driver.partition_scheme.boot_device(grub.version)
install_devices = [
d.name for d in self.driver.partition_scheme.parteds
if d.install_bootloader
]
grub.append_kernel_params('root=UUID=%s ' % mount2uuid['/'])
kernel = grub.kernel_name or \
gu.guess_kernel(chroot=chroot, regexp=grub.kernel_regexp)
initrd = grub.initrd_name or \
gu.guess_initrd(chroot=chroot, regexp=grub.initrd_regexp)
if grub.version == 1:
gu.grub1_cfg(kernel=kernel,
initrd=initrd,
kernel_params=grub.kernel_params,
chroot=chroot)
gu.grub1_install(install_devices, boot_device, chroot=chroot)
else:
# TODO(kozhukalov): implement which kernel to use by default
# Currently only grub1_cfg accepts kernel and initrd parameters.
gu.grub2_cfg(kernel_params=grub.kernel_params, chroot=chroot)
gu.grub2_install(install_devices, chroot=chroot)
# FIXME(agordeev) There's no convenient way to perfrom NIC remapping in
# Ubuntu, so injecting files prior the first boot should work
with open(chroot + '/etc/udev/rules.d/70-persistent-net.rules',
'w') as f:
f.write('# Generated by fuel-agent during provisioning: BEGIN\n')
# pattern is aa:bb:cc:dd:ee:ff_eth0,aa:bb:cc:dd:ee:ff_eth1
for mapping in self.driver.configdrive_scheme.\
common.udevrules.split(','):
mac_addr, nic_name = mapping.split('_')
f.write('SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", '
'ATTR{address}=="%s", ATTR{type}=="1", KERNEL=="eth*",'
' NAME="%s"\n' % (mac_addr, nic_name))
f.write('# Generated by fuel-agent during provisioning: END\n')
# FIXME(agordeev): Disable net-generator that will add new etries to
# 70-persistent-net.rules
with open(
chroot + '/etc/udev/rules.d/75-persistent-net-generator.rules',
'w') as f:
f.write('# Generated by fuel-agent during provisioning:\n'
'# DO NOT DELETE. It is needed to disable net-generator\n')
# FIXME(kozhukalov): Prevent nailgun-agent from doing anything.
# This ugly hack is to be used together with the command removing
# this lock file not earlier than /etc/rc.local
# The reason for this hack to appear is to prevent nailgun-agent from
# changing mcollective config at the same time when cloud-init
# does the same. Otherwise, we can end up with corrupted mcollective
# config. For details see https://bugs.launchpad.net/fuel/+bug/1449186
LOG.debug('Preventing nailgun-agent from doing '
'anything until it is unlocked')
utils.makedirs_if_not_exists(os.path.join(chroot, 'etc/nailgun-agent'))
with open(os.path.join(chroot, 'etc/nailgun-agent/nodiscover'), 'w'):
pass
with open(chroot + '/etc/fstab', 'wb') as f:
for fs in self.driver.partition_scheme.fss:
# TODO(kozhukalov): Think of improving the logic so as to
# insert a meaningful fsck order value which is last zero
# at fstab line. Currently we set it into 0 which means
# a corresponding file system will never be checked. We assume
# puppet or other configuration tool will care of it.
f.write('UUID=%s %s %s defaults 0 0\n' %
(mount2uuid[fs.mount], fs.mount, fs.type))
self.umount_target(chroot)
def do_bootloader(self):
LOG.debug('--- Installing bootloader (do_bootloader) ---')
chroot = '/tmp/target'
self.mount_target(chroot)
mount2uuid = {}
for fs in self.driver.partition_scheme.fss:
mount2uuid[fs.mount] = utils.execute(
'blkid',
'-o',
'value',
'-s',
'UUID',
fs.device,
check_exit_code=[0])[0].strip()
if '/' not in mount2uuid:
raise errors.WrongPartitionSchemeError(
'Error: device with / mountpoint has not been found')
grub = self.driver.grub
guessed_version = gu.guess_grub_version(chroot=chroot)
if guessed_version != grub.version:
grub.version = guessed_version
LOG.warning('Grub version differs from which the operating system '
'should have by default. Found version in image: '
'{0}'.format(guessed_version))
boot_device = self.driver.partition_scheme.boot_device(grub.version)
install_devices = [
d.name for d in self.driver.partition_scheme.parteds
if d.install_bootloader
]
grub.append_kernel_params('root=UUID=%s ' % mount2uuid['/'])
kernel = grub.kernel_name or \
gu.guess_kernel(chroot=chroot, regexp=grub.kernel_regexp)
initrd = grub.initrd_name or \
gu.guess_initrd(chroot=chroot, regexp=grub.initrd_regexp)
if grub.version == 1:
gu.grub1_cfg(kernel=kernel,
initrd=initrd,
kernel_params=grub.kernel_params,
chroot=chroot,
grub_timeout=CONF.grub_timeout)
gu.grub1_install(install_devices, boot_device, chroot=chroot)
else:
# TODO(kozhukalov): implement which kernel to use by default
# Currently only grub1_cfg accepts kernel and initrd parameters.
gu.grub2_cfg(kernel_params=grub.kernel_params,
chroot=chroot,
grub_timeout=CONF.grub_timeout)
gu.grub2_install(install_devices, chroot=chroot)
if CONF.fix_udev_net_rules:
# FIXME(agordeev) There's no convenient way to perfrom NIC
# remapping in Ubuntu, so injecting files prior the first boot
# should work
with open(chroot + '/etc/udev/rules.d/70-persistent-net.rules',
'wt',
encoding='utf-8') as f:
f.write(u'# Generated by fuel-agent during provisioning: '
u'BEGIN\n')
# pattern is aa:bb:cc:dd:ee:ff_eth0,aa:bb:cc:dd:ee:ff_eth1
for mapping in self.driver.configdrive_scheme.\
common.udevrules.split(','):
mac_addr, nic_name = mapping.split('_')
f.write(u'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", '
u'ATTR{address}=="%s", ATTR{type}=="1", '
u'KERNEL=="eth*", NAME="%s"\n' %
(mac_addr, nic_name))
f.write(
u'# Generated by fuel-agent during provisioning: END\n')
# FIXME(agordeev): Disable net-generator that will add new etries
# to 70-persistent-net.rules
with open(chroot +
'/etc/udev/rules.d/75-persistent-net-generator.rules',
'wt',
encoding='utf-8') as f:
f.write(u'# Generated by fuel-agent during provisioning:\n'
u'# DO NOT DELETE. It is needed to disable '
u'net-generator\n')
# FIXME(kozhukalov): Prevent nailgun-agent from doing anything.
# This ugly hack is to be used together with the command removing
# this lock file not earlier than /etc/rc.local
# The reason for this hack to appear is to prevent nailgun-agent from
# changing mcollective config at the same time when cloud-init
# does the same. Otherwise, we can end up with corrupted mcollective
# config. For details see https://bugs.launchpad.net/fuel/+bug/1449186
LOG.debug('Preventing nailgun-agent from doing '
'anything until it is unlocked')
utils.makedirs_if_not_exists(os.path.join(chroot, 'etc/nailgun-agent'))
with open(os.path.join(chroot, 'etc/nailgun-agent/nodiscover'), 'w'):
pass
# FIXME(kozhukalov): When we have just os-root fs image and don't have
# os-var-log fs image while / and /var/log are supposed to be
# separate file systems and os-var-log is mounted into
# non-empty directory on the / file system, those files in /var/log
# directory become unavailable.
# The thing is that among those file there is /var/log/upstart
# where upstart daemon writes its logs. We have specific upstart job
# which is to flush open files once all file systems are mounted.
# This job needs upstart directory to be available on os-var-log
# file system.
# This is just a temporary fix and correct fix will be available soon
# via updates.
utils.execute('mkdir', '-p', chroot + '/var/log/upstart')
with open(chroot + '/etc/fstab', 'wt', encoding='utf-8') as f:
for fs in self.driver.partition_scheme.fss:
# TODO(kozhukalov): Think of improving the logic so as to
# insert a meaningful fsck order value which is last zero
# at fstab line. Currently we set it into 0 which means
# a corresponding file system will never be checked. We assume
# puppet or other configuration tool will care of it.
if fs.mount == '/':
f.write(u'UUID=%s %s %s defaults,errors=panic 0 0\n' %
(mount2uuid[fs.mount], fs.mount, fs.type))
else:
f.write(u'UUID=%s %s %s defaults 0 0\n' %
(mount2uuid[fs.mount], fs.mount, fs.type))
self.umount_target(chroot)
def test_makedirs_if_not_exists_already_exists(
self, mock_isdir, mock_makedirs):
utils.makedirs_if_not_exists('/fake/path')
mock_isdir.assert_called_once_with('/fake/path')
self.assertEqual(mock_makedirs.mock_calls, [])
