def test_execute_ok_on_third_attempts(self, mock_popen, mock_sleep):
process = mock.Mock()
mock_popen.side_effect = [OSError, ValueError, process]
process.communicate.return_value = (None, None)
process.returncode = 0
utils.execute("/usr/bin/env", "false", attempts=3)
self.assertEqual(2 * [mock.call(CONF.execute_retry_delay)], mock_sleep.call_args_list)
def run_mksquashfs(chroot, output_name=None, compression_algorithm='xz'):
"""Pack the target system as squashfs using mksquashfs
:param chroot: chroot system, to be squashfs'd
:param output_name: output file name, might be a relative
or an absolute path
The kernel squashfs driver has to match with the user space squasfs tools.
Use the mksquashfs provided by the target distro to achieve this.
(typically the distro maintainers are smart enough to ship the correct
version of mksquashfs)
Use mksquashfs installed in the target system
1)Mount tmpfs under chroot/mnt
2)run mksquashfs inside a chroot
3)move result files to dstdir
"""
if not output_name:
output_name = 'root.squashfs' + six.text_type(uuid.uuid4())
utils.makedirs_if_not_exists(os.path.dirname(output_name))
dstdir = os.path.dirname(output_name)
temp = '.mksquashfs.tmp.' + six.text_type(uuid.uuid4())
s_dst = os.path.join(chroot, 'mnt/dst')
s_src = os.path.join(chroot, 'mnt/src')
try:
fu.mount_fs('tmpfs', 'mnt_{0}'.format(temp),
(os.path.join(chroot, 'mnt')),
'rw,nodev,nosuid,noatime,mode=0755,size=4M')
utils.makedirs_if_not_exists(s_src)
utils.makedirs_if_not_exists(s_dst)
# Bind mount the chroot to avoid including various temporary/virtual
# files (/proc, /sys, /dev, and so on) into the image
fu.mount_fs(None, chroot, s_src, opts='bind')
fu.mount_fs(None, None, s_src, 'remount,bind,ro')
fu.mount_fs(None, dstdir, s_dst, opts='bind')
# run mksquashfs
chroot_squash = os.path.join('/mnt/dst/' + temp)
long_squash = os.path.join(chroot, 'mnt/dst/{0}'.format(temp))
utils.execute('chroot',
chroot,
'mksquashfs',
'/mnt/src',
chroot_squash,
'-comp',
compression_algorithm,
'-no-progress',
'-noappend',
logged=True)
# move to result name
LOG.debug('Moving file: %s to: %s', long_squash, output_name)
shutil.move(long_squash, output_name)
except Exception as exc:
LOG.error('squashfs_image build failed: %s', exc)
raise
finally:
LOG.info('squashfs_image clean-up')
stop_chrooted_processes(chroot, signal=signal.SIGTERM)
fu.umount_fs(os.path.join(chroot, 'mnt/dst'))
fu.umount_fs(os.path.join(chroot, 'mnt/src'))
fu.umount_fs(os.path.join(chroot, 'mnt'))
def mdremove(mdname):
# check if md exists
if mdname not in get_mdnames():
raise errors.MDNotFoundError(
'Error while removing md: md %s not found' % mdname)
# FIXME: The issue faced was quiet hard to reproduce and to figure out the
# root cause. For unknown reason already removed md device is
# unexpectedly returning back after a while from time to time making
# new md device creation to fail.
# Still the actual reason of its failure is unknown, but after a
# searching on a web a mention was found about a race in udev
# http://dev.bizo.com/2012/07/mdadm-device-or-resource-busy.html
# The article recommends to disable udev's queue entirely during md
# device manipulation which sounds rather unappropriate for our case.
# And the link to original post on mailing list suggests to execute
# `udevadm settle` before removing the md device.
# here -> http://permalink.gmane.org/gmane.linux.raid/34027
# So, what was done. `udevadm settle` calls were placed just
# before any of `mdadm` calls and the analizyng the logs was started.
# According to the manual `settle` is an option that "Watches the
# udev event queue, and exits if all current events are handled".
# That means it will wait for udev's finishing of processing the
# events. According to the logs noticeable delay had been recognized
# between `udevadm settle` and the next `mdadm` call.
# The delay was about 150-200ms or even bigger. It was appeared
# right before the `mdadm --stop` call. That just means that udev was
# too busy with events when we start to modifiy md devices hard.
# Thus `udevadm settle` is helping to avoid the later failure and
# to prevent strange behaviour of md device.
utils.udevadm_settle()
utils.execute('mdadm', '--stop', mdname, check_exit_code=[0])
utils.execute('mdadm', '--remove', mdname, check_exit_code=[0, 1])
def _find_hw_fstab(self):
mount_dir = '/mnt'
fstabs = []
fstab_fss = filter(lambda fss: fss.mount == '/',
self.partition_scheme.fss)
for fss in fstab_fss:
fss_dev = fss.device
fstab_path = os.path.join(mount_dir, 'etc', 'fstab')
try:
utils.execute('mount',
fss_dev,
mount_dir,
run_as_root=True,
check_exit_code=[0])
fstab, _ = utils.execute('cat',
fstab_path,
run_as_root=True,
check_exit_code=[0])
utils.execute('umount', mount_dir, run_as_root=True)
except errors.ProcessExecutionError as e:
raise errors.HardwarePartitionSchemeCannotBeReadError(
"Cannot read fstab from %s partition. Error occurred: %s" %
(fss_dev, str(e)))
LOG.info("fstab has been found on %s:\n%s" % (fss_dev, fstab))
fstabs.append(fstab)
return '\n'.join(fstabs)
def test_check_exit_code_boolean(self):
utils.execute('/usr/bin/env', 'false', check_exit_code=False)
self.assertRaises(errors.ProcessExecutionError,
utils.execute,
'/usr/bin/env',
'false',
check_exit_code=True)
def _find_hw_fstab(self):
mount_dir = '/mnt'
fstabs = []
fstab_fss = filter(lambda fss: fss.mount == '/',
self.partition_scheme.fss)
for fss in fstab_fss:
fss_dev = fss.device
fstab_path = os.path.join(mount_dir, 'etc', 'fstab')
try:
utils.execute('mount', fss_dev, mount_dir, run_as_root=True,
check_exit_code=[0])
fstab, _ = utils.execute('cat', fstab_path,
run_as_root=True,
check_exit_code=[0])
utils.execute('umount', mount_dir, run_as_root=True)
except errors.ProcessExecutionError as e:
raise errors.HardwarePartitionSchemeCannotBeReadError(
"Cannot read fstab from %s partition. Error occurred: %s"
% (fss_dev, str(e))
)
LOG.info("fstab has been found on %s:\n%s" % (fss_dev, fstab))
fstabs.append(fstab)
return '\n'.join(fstabs)
def recompress_initramfs(chroot, compress='xz', initrd_mask='initrd*'):
"""Remove old and rebuild initrd
:param chroot:
:param compress: compression type for initrd
:return:
:initrd_mask: search kernel file by Unix style pathname
"""
env_vars = copy.deepcopy(os.environ)
add_env_vars = {'TMPDIR': '/tmp',
'TMP': '/tmp'}
LOG.info('Changing initramfs compression type to: %s', compress)
utils.execute(
'sed', '-i', 's/^COMPRESS=.*/COMPRESS={0}/'.format(compress),
os.path.join(chroot, 'etc/initramfs-tools/initramfs.conf'))
boot_dir = os.path.join(chroot, 'boot')
initrds = glob.glob(os.path.join(boot_dir, initrd_mask))
LOG.debug('Removing initrd images: %s', initrds)
remove_files('/', initrds)
env_vars.update(add_env_vars)
cmds = ['chroot', chroot, 'update-initramfs -v -c -k all']
utils.execute(*cmds,
env_variables=env_vars, logged=True)
LOG.debug('Running "update-initramfs" completed')
def override_lvm_config_value(chroot, section, name, value, lvm_conf_file):
"""Override option in LVM configuration.
If option is not valid, then errors.ProcessExecutionError will be raised
and lvm configuration will remain unchanged
"""
lvm_conf_file = os.path.join(chroot, lvm_conf_file.lstrip('/'))
updated_config = _update_option_in_lvm_raw_config(
section, name, value,
utils.execute('chroot', chroot, 'lvm dumpconfig')[0])
lvm_conf_file_bak = '{}.bak.{}'.format(lvm_conf_file,
time.strftime("%Y_%m_%d_%H_%M_%S"))
shutil.copy(lvm_conf_file, lvm_conf_file_bak)
LOG.debug('Backup for origin LVM configuration file: {}'
''.format(lvm_conf_file_bak))
with open(lvm_conf_file, mode='w') as lvm_conf:
lvm_conf.write(updated_config)
# NOTE(sslypushenko) Extra cycle of dump/save lvm.conf is required to be
# sure that updated configuration is valid and to adjust it to general
# lvm.conf formatting
try:
current_config = utils.execute('chroot', chroot, 'lvm dumpconfig')[0]
with open(lvm_conf_file, mode='w') as lvm_conf:
lvm_conf.write(current_config)
LOG.info('LVM configuration {} updated. '
'Option {}/{} gets new value: {}'
''.format(lvm_conf_file, section, name, value))
except errors.ProcessExecutionError as exc:
shutil.move(lvm_conf_file_bak, lvm_conf_file)
LOG.debug('Option {}/{} can not be updated with value {}. '
'Configuration restored'.format(section, name, value))
raise exc
def _make_configdrive_image(self, src_files):
bs = 4096
configdrive_device = self.driver.partition_scheme.configdrive_device()
size = utils.execute('blockdev', '--getsize64', configdrive_device)[0]
size = int(size.strip())
utils.execute('truncate', '--size=%d' % size, CONF.config_drive_path)
fu.make_fs(
fs_type='ext2',
fs_options=' -b %d -F ' % bs,
fs_label='config-2',
dev=six.text_type(CONF.config_drive_path))
mount_point = tempfile.mkdtemp(dir=CONF.tmp_path)
try:
fu.mount_fs('ext2', CONF.config_drive_path, mount_point)
for file_path in src_files:
name = os.path.basename(file_path)
if os.path.isdir(file_path):
shutil.copytree(file_path, os.path.join(mount_point, name))
else:
shutil.copy2(file_path, mount_point)
except Exception as exc:
LOG.error('Error copying files to configdrive: %s', exc)
raise
finally:
fu.umount_fs(mount_point)
os.rmdir(mount_point)
def mdremove(mdname):
# check if md exists
if mdname not in get_mdnames():
raise errors.MDNotFoundError(
'Error while removing md: md %s not found' % mdname)
# FIXME: The issue faced was quiet hard to reproduce and to figure out the
# root cause. For unknown reason already removed md device is
# unexpectedly returning back after a while from time to time making
# new md device creation to fail.
# Still the actual reason of its failure is unknown, but after a
# searching on a web a mention was found about a race in udev
# http://dev.bizo.com/2012/07/mdadm-device-or-resource-busy.html
# The article recommends to disable udev's queue entirely during md
# device manipulation which sounds rather unappropriate for our case.
# And the link to original post on mailing list suggests to execute
# `udevadm settle` before removing the md device.
# here -> http://permalink.gmane.org/gmane.linux.raid/34027
# So, what was done. `udevadm settle` calls were placed just
# before any of `mdadm` calls and the analizyng the logs was started.
# According to the manual `settle` is an option that "Watches the
# udev event queue, and exits if all current events are handled".
# That means it will wait for udev's finishing of processing the
# events. According to the logs noticeable delay had been recognized
# between `udevadm settle` and the next `mdadm` call.
# The delay was about 150-200ms or even bigger. It was appeared
# right before the `mdadm --stop` call. That just means that udev was
# too busy with events when we start to modifiy md devices hard.
# Thus `udevadm settle` is helping to avoid the later failure and
# to prevent strange behaviour of md device.
utils.udevadm_settle()
utils.execute('mdadm', '--stop', mdname, check_exit_code=[0])
utils.execute('mdadm', '--remove', mdname, check_exit_code=[0, 1])
def vgcreate(vgname, pvname, *args):
# check if vg already exists
if get_first_by_key_value(vgdisplay(), 'name', vgname, False):
raise errors.VGAlreadyExistsError(
'Error while creating vg: vg %s already exists' % vgname)
pvnames = [pvname] + list(args)
_vg_attach_validate(pvnames)
utils.execute('vgcreate', vgname, *pvnames, check_exit_code=[0])
def vgextend(vgname, pvname, *args):
# check if vg exists
if not get_first_by_key_value(vgdisplay(), 'name', vgname, False):
raise errors.VGNotFoundError(
'Error while extending vg: vg %s not found' % vgname)
pvnames = [pvname] + list(args)
_vg_attach_validate(pvnames)
utils.execute('vgextend', vgname, *pvnames, check_exit_code=[0])
def vgcreate(vgname, pvname, *args):
# check if vg already exists
if get_first_by_key_value(vgdisplay(), 'name', vgname, False):
raise errors.VGAlreadyExistsError(
'Error while creating vg: vg %s already exists' % vgname)
pvnames = [pvname] + list(args)
_vg_attach_validate(pvnames)
utils.execute('vgcreate', vgname, *pvnames, check_exit_code=[0])
def test_execute_ok_on_third_attempts(self, mock_popen, mock_sleep):
process = mock.Mock()
mock_popen.side_effect = [OSError, ValueError, process]
process.communicate.return_value = (None, None)
process.returncode = 0
utils.execute('/usr/bin/env', 'false', attempts=3)
self.assertEqual(2 * [mock.call(CONF.execute_retry_delay)],
mock_sleep.call_args_list)
def vgextend(vgname, pvname, *args):
# check if vg exists
if not get_first_by_key_value(vgdisplay(), 'name', vgname, False):
raise errors.VGNotFoundError(
'Error while extending vg: vg %s not found' % vgname)
pvnames = [pvname] + list(args)
_vg_attach_validate(pvnames)
utils.execute('vgextend', vgname, *pvnames, check_exit_code=[0])
def mount_bind(chroot, path, path2=None):
if not path2:
path2 = path
utils.execute('mount',
'--bind',
path,
os.path.join(chroot, path2.strip(os.sep)),
check_exit_code=[0])
def shrink_sparse_file(filename):
"""Shrinks file to its size of actual data. Only ext fs are supported."""
utils.execute('e2fsck', '-y', '-f', filename)
utils.execute('resize2fs', '-M', filename)
data = hu.parse_simple_kv('dumpe2fs', filename)
block_count = int(data['block count'])
block_size = int(data['block size'])
with open(filename, 'rwb+') as f:
f.truncate(block_count * block_size)
def create_sparse_tmp_file(dir, suffix, size=8192):
"""Creates sparse file.
Creates file which consumes disk space more efficiently when the file
itself is mostly empty.
"""
tf = tempfile.NamedTemporaryFile(dir=dir, suffix=suffix, delete=False)
utils.execute('truncate', '-s', '%sM' % size, tf.name)
return tf.name
def pvcreate(pvname, metadatasize=64, metadatacopies=2):
# check if pv already exists
if get_first_by_key_value(pvdisplay(), 'name', pvname, False):
raise errors.PVAlreadyExistsError(
'Error while creating pv: pv %s already exists' % pvname)
utils.execute('pvcreate',
'--metadatacopies', str(metadatacopies),
'--metadatasize', str(metadatasize) + 'm',
pvname, check_exit_code=[0])
def shrink_sparse_file(filename):
"""Shrinks file to its size of actual data. Only ext fs are supported."""
utils.execute('e2fsck', '-y', '-f', filename)
utils.execute('resize2fs', '-M', filename)
data = hu.parse_simple_kv('dumpe2fs', filename)
block_count = int(data['block count'])
block_size = int(data['block size'])
with open(filename, 'rwb+') as f:
f.truncate(block_count * block_size)
def create_sparse_tmp_file(dir, suffix, size=8192):
"""Creates sparse file.
Creates file which consumes disk space more efficiently when the file
itself is mostly empty.
"""
tf = tempfile.NamedTemporaryFile(dir=dir, suffix=suffix, delete=False)
utils.execute('truncate', '-s', '%sM' % size, tf.name)
return tf.name
def copy_update_certs(certs, chroot):
"""Try to copy and update CA certificates in chroot"""
for cert in certs:
rsync_inject(cert, chroot)
utils.execute('chroot',
chroot,
'update-ca-certificates',
check_exit_code=False,
logged=True)
def run_mksquashfs(chroot, output_name=None, compression_algorithm='xz'):
"""Pack the target system as squashfs using mksquashfs
:param chroot: chroot system, to be squashfs'd
:param output_name: output file name, might be a relative
or an absolute path
The kernel squashfs driver has to match with the user space squasfs tools.
Use the mksquashfs provided by the target distro to achieve this.
(typically the distro maintainers are smart enough to ship the correct
version of mksquashfs)
Use mksquashfs installed in the target system
1)Mount tmpfs under chroot/mnt
2)run mksquashfs inside a chroot
3)move result files to dstdir
"""
if not output_name:
output_name = 'root.squashfs' + six.text_type(uuid.uuid4())
utils.makedirs_if_not_exists(os.path.dirname(output_name))
dstdir = os.path.dirname(output_name)
temp = '.mksquashfs.tmp.' + six.text_type(uuid.uuid4())
s_dst = os.path.join(chroot, 'mnt/dst')
s_src = os.path.join(chroot, 'mnt/src')
try:
fu.mount_fs(
'tmpfs', 'mnt_{0}'.format(temp),
(os.path.join(chroot, 'mnt')),
'rw,nodev,nosuid,noatime,mode=0755,size=4M')
utils.makedirs_if_not_exists(s_src)
utils.makedirs_if_not_exists(s_dst)
# Bind mount the chroot to avoid including various temporary/virtual
# files (/proc, /sys, /dev, and so on) into the image
fu.mount_fs(None, chroot, s_src, opts='bind')
fu.mount_fs(None, None, s_src, 'remount,bind,ro')
fu.mount_fs(None, dstdir, s_dst, opts='bind')
# run mksquashfs
chroot_squash = os.path.join('/mnt/dst/' + temp)
long_squash = os.path.join(chroot, 'mnt/dst/{0}'.format(temp))
utils.execute(
'chroot', chroot, 'mksquashfs', '/mnt/src',
chroot_squash,
'-comp', compression_algorithm,
'-no-progress', '-noappend', logged=True)
# move to result name
LOG.debug('Moving file: %s to: %s', long_squash, output_name)
shutil.move(long_squash, output_name)
except Exception as exc:
LOG.error('squashfs_image build failed: %s', exc)
raise
finally:
LOG.info('squashfs_image clean-up')
stop_chrooted_processes(chroot, signal=signal.SIGTERM)
fu.umount_fs(os.path.join(chroot, 'mnt/dst'))
fu.umount_fs(os.path.join(chroot, 'mnt/src'))
fu.umount_fs(os.path.join(chroot, 'mnt'))
def deattach_loop(loop, check_exit_code=[0]):
LOG.debug('Trying to figure out if loop device %s is attached', loop)
output = utils.execute('losetup', '-a')[0]
for line in output.split('\n'):
# output lines are assumed to have the following format
# /dev/loop0: [fd03]:130820 (/dev/loop0)
if loop == line.split(':')[0]:
LOG.debug('Loop device %s seems to be attached. '
'Trying to detach.', loop)
utils.execute('losetup', '-d', loop,
check_exit_code=check_exit_code)
def grub2_install(install_devices, chroot='', boot_root='', lvm_boot=False):
grub_install = guess_grub_install(chroot=chroot)
for install_device in install_devices:
cmd = [grub_install, install_device]
if lvm_boot:
cmd.append('--modules="lvm"')
if boot_root:
cmd.append('--boot-directory={}/boot'.format(boot_root))
elif chroot:
cmd[:0] = ['chroot', chroot]
utils.execute(*cmd, run_as_root=True, check_exit_code=[0])
def grub2_install(install_devices, chroot='', boot_root='', lvm_boot=False):
grub_install = guess_grub_install(chroot=chroot)
for install_device in install_devices:
cmd = [grub_install, install_device]
if lvm_boot:
cmd.append('--modules="lvm"')
if boot_root:
cmd.append('--boot-directory={}/boot'.format(boot_root))
elif chroot:
cmd[:0] = ['chroot', chroot]
utils.execute(*cmd, run_as_root=True, check_exit_code=[0])
def pvremove(pvname):
pv = get_first_by_key_value(pvdisplay(), 'name', pvname)
# check if pv exists
if not pv:
raise errors.PVNotFoundError(
'Error while removing pv: pv %s not found' % pvname)
# check if pv is attached to some vg
if pv['vg'] is not None:
raise errors.PVBelongsToVGError('Error while removing pv: '
'pv belongs to vg %s' % pv['vg'])
utils.execute('pvremove', '-ff', '-y', pvname, check_exit_code=[0])
def pvremove(pvname):
pv = get_first_by_key_value(pvdisplay(), 'name', pvname)
# check if pv exists
if not pv:
raise errors.PVNotFoundError(
'Error while removing pv: pv %s not found' % pvname)
# check if pv is attached to some vg
if pv['vg'] is not None:
raise errors.PVBelongsToVGError('Error while removing pv: '
'pv belongs to vg %s' % pv['vg'])
utils.execute('pvremove', '-ff', '-y', pvname, check_exit_code=[0])
def pvcreate(pvname, metadatasize=64, metadatacopies=2):
# check if pv already exists
if get_first_by_key_value(pvdisplay(), 'name', pvname, False):
raise errors.PVAlreadyExistsError(
'Error while creating pv: pv %s already exists' % pvname)
utils.execute('pvcreate',
'--metadatacopies',
str(metadatacopies),
'--metadatasize',
str(metadatasize) + 'm',
pvname,
check_exit_code=[0])
def vgreduce(vgname, pvname, *args):
# check if vg exists
if not get_first_by_key_value(vgdisplay(), 'name', vgname, False):
raise errors.VGNotFoundError(
'Error while reducing vg: vg %s not found' % vgname)
pvnames = [pvname] + list(args)
# check if all necessary pv are attached to vg
if not set(pvnames).issubset(
set([pv['name'] for pv in pvdisplay() if pv['vg'] == vgname])):
raise errors.PVNotFoundError(
'Error while reducing vg: at least one of pv is '
'not attached to vg')
utils.execute('vgreduce', '-f', vgname, *pvnames, check_exit_code=[0])
def vgreduce(vgname, pvname, *args):
# check if vg exists
if not get_first_by_key_value(vgdisplay(), 'name', vgname, False):
raise errors.VGNotFoundError(
'Error while reducing vg: vg %s not found' % vgname)
pvnames = [pvname] + list(args)
# check if all necessary pv are attached to vg
if not set(pvnames).issubset(
set([pv['name'] for pv in pvdisplay() if pv['vg'] == vgname])):
raise errors.PVNotFoundError(
'Error while reducing vg: at least one of pv is '
'not attached to vg')
utils.execute('vgreduce', '-f', vgname, *pvnames, check_exit_code=[0])
def do_post_inst(chroot,
allow_unsigned_file='allow_unsigned_packages',
force_ipv4_file='force_ipv4'):
# NOTE(agordeev): set up password for root
utils.execute('sed', '-i', 's%root:[\*,\!]%root:' + ROOT_PASSWORD + '%',
os.path.join(chroot, 'etc/shadow'))
# NOTE(agordeev): backport from bash-script:
# in order to prevent the later puppet workflow outage, puppet service
# should be disabled on a node startup.
# Being enabled by default, sometimes it leads to puppet service hanging
# and recognizing the deployment as failed.
# TODO(agordeev): take care of puppet service for other distros, once
# bareon will be capable of building images for them too.
if os.path.exists(os.path.join(chroot, 'etc/init.d/puppet')):
utils.execute('chroot', chroot, 'update-rc.d', 'puppet', 'disable')
# NOTE(agordeev): disable mcollective to be automatically started on boot
# to prevent confusing messages in its log (regarding connection errors).
with open(os.path.join(chroot, 'etc/init/mcollective.override'), 'w') as f:
f.write("manual\n")
# NOTE(agordeev): remove custom policy-rc.d which is needed to disable
# execution of post/pre-install package hooks and start of services
remove_files(chroot, ['usr/sbin/policy-rc.d'])
# enable mdadm (remove nomdadmddf nomdadmism options from cmdline)
utils.execute('chroot', chroot, 'dpkg-divert', '--local', '--add',
os.path.join('/', GRUB2_DMRAID_SETTINGS))
remove_files(chroot, [GRUB2_DMRAID_SETTINGS])
# remove cached apt files
utils.execute('chroot', chroot, 'apt-get', 'clean')
clean_apt_settings(chroot,
allow_unsigned_file=allow_unsigned_file,
force_ipv4_file=force_ipv4_file)
def do_post_inst(chroot, allow_unsigned_file='allow_unsigned_packages',
force_ipv4_file='force_ipv4'):
# NOTE(agordeev): set up password for root
utils.execute('sed', '-i',
's%root:[\*,\!]%root:' + ROOT_PASSWORD + '%',
os.path.join(chroot, 'etc/shadow'))
# NOTE(agordeev): backport from bash-script:
# in order to prevent the later puppet workflow outage, puppet service
# should be disabled on a node startup.
# Being enabled by default, sometimes it leads to puppet service hanging
# and recognizing the deployment as failed.
# TODO(agordeev): take care of puppet service for other distros, once
# bareon will be capable of building images for them too.
if os.path.exists(os.path.join(chroot, 'etc/init.d/puppet')):
utils.execute('chroot', chroot, 'update-rc.d', 'puppet', 'disable')
# NOTE(agordeev): disable mcollective to be automatically started on boot
# to prevent confusing messages in its log (regarding connection errors).
with open(os.path.join(chroot, 'etc/init/mcollective.override'), 'w') as f:
f.write("manual\n")
# NOTE(agordeev): remove custom policy-rc.d which is needed to disable
# execution of post/pre-install package hooks and start of services
remove_files(chroot, ['usr/sbin/policy-rc.d'])
# enable mdadm (remove nomdadmddf nomdadmism options from cmdline)
utils.execute('chroot', chroot, 'dpkg-divert', '--local', '--add',
os.path.join('/', GRUB2_DMRAID_SETTINGS))
remove_files(chroot, [GRUB2_DMRAID_SETTINGS])
# remove cached apt files
utils.execute('chroot', chroot, 'apt-get', 'clean')
clean_apt_settings(chroot, allow_unsigned_file=allow_unsigned_file,
force_ipv4_file=force_ipv4_file)
def deattach_loop(loop, check_exit_code=[0]):
LOG.debug('Trying to figure out if loop device %s is attached', loop)
output = utils.execute('losetup', '-a')[0]
for line in output.split('\n'):
# output lines are assumed to have the following format
# /dev/loop0: [fd03]:130820 (/dev/loop0)
if loop == line.split(':')[0]:
LOG.debug(
'Loop device %s seems to be attached. '
'Trying to detach.', loop)
utils.execute('losetup',
'-d',
loop,
check_exit_code=check_exit_code)
def do_copyimage(self, os_id):
os_path = '/tmp/target/'
with self.mount_target(os_path, os_id, pseudo=False,
treat_mtab=False):
for image in self.driver.image_scheme.get_os_images(os_id):
target_image_path = os.path.join(os_path,
image.target_device.strip(
os.sep))
LOG.debug('Starting rsync from %s to %s', image.uri,
target_image_path)
rsync_flags = image.deployment_flags.get('rsync_flags',
'-a -A -X')
utils.execute('rsync', rsync_flags, image.uri,
target_image_path, check_exit_code=[0])
def make_targz(source_dir, output_name=None):
"""Archive the given directory
:param source_dir: directory to archive
:param output_name: output file name, might be a relative
or an absolute path
"""
if not output_name:
output_name = six.text_type(uuid.uuid4()) + '.tar.gz'
utils.makedirs_if_not_exists(os.path.dirname(output_name))
LOG.info('Creating archive: %s', output_name)
utils.execute('tar', '-czf', output_name, '--directory',
os.path.normcase(source_dir), '.', logged=True)
return output_name
def grub2_cfg_bundled(kernel_params='', chroot='', grub_timeout=10,
lvm_boot=False):
# NOTE(oberezovskyi): symlink is required because of grub2-probe fails
# to find device with root partition of fuel agent.
# It's actuall in the ram and "device" is "rootfs"
os.symlink(chroot, '/tmp/rootfs')
with grub2_prepare(kernel_params, chroot, grub_timeout, lvm_boot):
# NOTE(oberezovskyi): required to prevent adding boot entries for
# ramdisk
os.remove('/etc/grub.d/10_linux')
cmd = [guess_grub2_mkconfig(), '-o', chroot + '/boot/grub2/grub.cfg']
utils.execute(*cmd, run_as_root=True, cwd='/tmp/')
os.remove('/tmp/rootfs')
def parse_dmidecode(type):
"""Parses `dmidecode` output.
:param type: A string with type of entity to display.
:returns: A list with dictionaries of entities for specified type.
"""
output = utils.execute('dmidecode', '-q', '--type', type)
lines = output[0].split('\n')
info = []
multiline_values = None
section = 0
for line in lines:
if len(line) != 0 and len(line.strip()) == len(line):
info.append({})
section = len(info) - 1
try:
k, v = (l.strip() for l in line.split(':', 1))
except ValueError:
k = line.strip()
if not k:
multiline_values = None
if multiline_values:
info[section][multiline_values].append(k)
else:
if not v:
multiline_values = k.lower()
info[section][multiline_values] = []
else:
info[section][k.lower()] = v
return info
def mount_target(self, chroot, treat_mtab=True, pseudo=True):
"""Mount a set of file systems into a chroot
:param chroot: Directory where to mount file systems
:param treat_mtab: If mtab needs to be actualized (Default: True)
:param pseudo: If pseudo file systems
need to be mounted (Default: True)
"""
LOG.debug('Mounting target file systems: %s', chroot)
# Here we are going to mount all file systems in partition scheme.
for fs in self.driver.partition_scheme.fs_sorted_by_depth():
if fs.mount == 'swap':
continue
mount = chroot + fs.mount
utils.makedirs_if_not_exists(mount)
fu.mount_fs(fs.type, str(fs.device), mount)
if pseudo:
for path in ('/sys', '/dev', '/proc'):
utils.makedirs_if_not_exists(chroot + path)
fu.mount_bind(chroot, path)
if treat_mtab:
mtab = utils.execute('chroot', chroot, 'grep', '-v', 'rootfs',
'/proc/mounts')[0]
mtab_path = chroot + '/etc/mtab'
if os.path.islink(mtab_path):
os.remove(mtab_path)
with open(mtab_path, 'wt', encoding='utf-8') as f:
f.write(six.text_type(mtab))
def set_partition_flag(dev, num, flag, state='on'):
"""Sets flag on a partition
:param dev: A device file, e.g. /dev/sda.
:param num: Partition number
:param flag: Flag name. Must be one of 'bios_grub', 'legacy_boot',
'boot', 'raid', 'lvm'
:param state: Desiable flag state. 'on' or 'off'. Default is 'on'.
:returns: None
"""
LOG.debug('Trying to set partition flag: dev=%s num=%s flag=%s state=%s' %
(dev, num, flag, state))
# parted supports more flags but we are interested in
# setting only this subset of them.
# not all of these flags are compatible with one another.
if flag not in ('bios_grub', 'legacy_boot', 'boot', 'raid', 'lvm'):
raise errors.WrongPartitionSchemeError(
'Unsupported partition flag: %s' % flag)
if state not in ('on', 'off'):
raise errors.WrongPartitionSchemeError(
'Wrong partition flag state: %s' % state)
utils.udevadm_settle()
out, err = utils.execute('parted', '-s', dev, 'set', str(num),
flag, state, check_exit_code=[0, 1])
LOG.debug('Parted output: \n%s' % out)
reread_partitions(dev, out=out)
def udevreport(dev):
"""Builds device udevadm report.
:param dev: A device file, e.g. /dev/sda.
:returns: A dict of udev device properties.
"""
report = utils.execute('udevadm',
'info',
'--query=property',
'--export',
'--name={0}'.format(dev),
check_exit_code=[0])[0]
spec = {}
for line in [l for l in report.splitlines() if l]:
key, value = line.split('=', 1)
value = value.strip('\'')
# This is a list of symbolic links which were created for this
# block device (e.g. /dev/disk/by-id/foobar)
if key == 'DEVLINKS':
spec['DEVLINKS'] = value.split()
if key in UDEV_PROPERTIES:
spec[key] = value
return spec
def run_debootstrap(uri, suite, chroot, arch='amd64', eatmydata=False,
attempts=10, proxies=None, direct_repo_addr=None):
"""Builds initial base system.
debootstrap builds initial base system which is capable to run apt-get.
debootstrap is well known for its glithcy resolving of package dependecies,
so the rest of packages will be installed later by run_apt_get.
"""
env_vars = copy.deepcopy(os.environ)
for proto in six.iterkeys(PROXY_PROTOCOLS):
if proto in (proxies or {}):
LOG.debug('Using {0} proxy {1} for debootstrap'.format(
proto, proxies[proto]))
env_vars['{0}_proxy'.format(proto)] = proxies[proto]
if direct_repo_addr:
env_vars['no_proxy'] = ','.join(direct_repo_addr)
LOG.debug('Setting no_proxy for: {0}'.format(env_vars['no_proxy']))
cmds = ['debootstrap',
'--include={0}'.format(",".join(ADDITIONAL_DEBOOTSTRAP_PACKAGES)),
'--verbose', '--no-check-gpg',
'--arch={0}'.format(arch)]
if eatmydata:
cmds.extend(['--include=eatmydata'])
cmds.extend([suite, chroot, uri])
stdout, stderr = utils.execute(*cmds, attempts=attempts,
env_variables=env_vars)
LOG.debug('Running deboostrap completed.\nstdout: %s\nstderr: %s', stdout,
stderr)
def mount_target(self, chroot, treat_mtab=True, pseudo=True):
"""Mount a set of file systems into a chroot
:param chroot: Directory where to mount file systems
:param treat_mtab: If mtab needs to be actualized (Default: True)
:param pseudo: If pseudo file systems
need to be mounted (Default: True)
"""
LOG.debug('Mounting target file systems: %s', chroot)
# Here we are going to mount all file systems in partition scheme.
for fs in self.driver.partition_scheme.fs_sorted_by_depth():
if fs.mount == 'swap':
continue
mount = chroot + fs.mount
utils.makedirs_if_not_exists(mount)
fu.mount_fs(fs.type, str(fs.device), mount)
if pseudo:
for path in ('/sys', '/dev', '/proc'):
utils.makedirs_if_not_exists(chroot + path)
fu.mount_bind(chroot, path)
if treat_mtab:
mtab = utils.execute(
'chroot', chroot, 'grep', '-v', 'rootfs', '/proc/mounts')[0]
mtab_path = chroot + '/etc/mtab'
if os.path.islink(mtab_path):
os.remove(mtab_path)
with open(mtab_path, 'wt', encoding='utf-8') as f:
f.write(six.text_type(mtab))
def parse_dmidecode(type):
"""Parses `dmidecode` output.
:param type: A string with type of entity to display.
:returns: A list with dictionaries of entities for specified type.
"""
output = utils.execute('dmidecode', '-q', '--type', type)
lines = output[0].split('\n')
info = []
multiline_values = None
section = 0
for line in lines:
if len(line) != 0 and len(line.strip()) == len(line):
info.append({})
section = len(info) - 1
try:
k, v = (l.strip() for l in line.split(':', 1))
except ValueError:
k = line.strip()
if not k:
multiline_values = None
if multiline_values:
info[section][multiline_values].append(k)
else:
if not v:
multiline_values = k.lower()
info[section][multiline_values] = []
else:
info[section][k.lower()] = v
return info
def udevreport(dev):
"""Builds device udevadm report.
:param dev: A device file, e.g. /dev/sda.
:returns: A dict of udev device properties.
"""
report = utils.execute('udevadm',
'info',
'--query=property',
'--export',
'--name={0}'.format(dev),
check_exit_code=[0])[0]
spec = {}
for line in [l for l in report.splitlines() if l]:
key, value = line.split('=', 1)
value = value.strip('\'')
# This is a list of symbolic links which were created for this
# block device (e.g. /dev/disk/by-id/foobar)
if key == 'DEVLINKS':
spec['DEVLINKS'] = value.split()
if key in UDEV_PROPERTIES:
spec[key] = value
return spec
def make_partition(dev, begin, end, ptype):
LOG.debug('Trying to create a partition: dev=%s begin=%s end=%s' %
(dev, begin, end))
if ptype not in ('primary', 'logical'):
raise errors.WrongPartitionSchemeError(
'Wrong partition type: %s' % ptype)
# check begin >= end
if begin >= end:
raise errors.WrongPartitionSchemeError(
'Wrong boundaries: begin >= end')
# check if begin and end are inside one of free spaces available
if not any(x['fstype'] == 'free' and begin >= x['begin'] and
end <= x['end'] for x in info(dev)['parts']):
raise errors.WrongPartitionSchemeError(
'Invalid boundaries: begin and end '
'are not inside available free space')
utils.udevadm_settle()
out, err = utils.execute(
'parted', '-a', 'optimal', '-s', dev, 'unit', 'MiB',
'mkpart', ptype, str(begin), str(end), check_exit_code=[0, 1])
LOG.debug('Parted output: \n%s' % out)
reread_partitions(dev, out=out)
def get_installed_packages(chroot):
"""The packages installed in chroot along with their versions"""
out, err = utils.execute('chroot', chroot, 'dpkg-query', '-W',
'-f="${Package} ${Version};;"')
pkglist = filter(None, out.split(';;'))
return dict([pkgver.split() for pkgver in pkglist])
def _mount_target(self,
mount_dir,
os_id=None,
pseudo=True,
treat_mtab=True):
LOG.debug('Mounting target file systems: %s', mount_dir)
# Here we are going to mount all file systems in partition schema.
for fs in self.driver.partition_scheme.fs_sorted_by_depth(os_id):
if fs.mount == 'swap':
continue
mount = os.path.join(mount_dir, fs.mount.strip(os.sep))
utils.makedirs_if_not_exists(mount)
fu.mount_fs(fs.type, str(fs.device), mount)
if pseudo:
for path in ('/sys', '/dev', '/proc'):
utils.makedirs_if_not_exists(
os.path.join(mount_dir, path.strip(os.sep)))
fu.mount_bind(mount_dir, path)
if treat_mtab:
mtab = utils.execute('chroot', mount_dir, 'grep', '-v', 'rootfs',
'/proc/mounts')[0]
mtab_path = os.path.join(mount_dir, 'etc/mtab')
if os.path.islink(mtab_path):
os.remove(mtab_path)
with open(mtab_path, 'wt', encoding='utf-8') as f:
f.write(six.text_type(mtab))
def set_partition_flag(dev, num, flag, state='on'):
"""Sets flag on a partition
:param dev: A device file, e.g. /dev/sda.
:param num: Partition number
:param flag: Flag name. Must be one of 'bios_grub', 'legacy_boot',
'boot', 'raid', 'lvm'
:param state: Desiable flag state. 'on' or 'off'. Default is 'on'.
:returns: None
"""
LOG.debug('Trying to set partition flag: dev=%s num=%s flag=%s state=%s' %
(dev, num, flag, state))
# parted supports more flags but we are interested in
# setting only this subset of them.
# not all of these flags are compatible with one another.
if flag not in ('bios_grub', 'legacy_boot', 'boot', 'raid', 'lvm'):
raise errors.WrongPartitionSchemeError(
'Unsupported partition flag: %s' % flag)
if state not in ('on', 'off'):
raise errors.WrongPartitionSchemeError(
'Wrong partition flag state: %s' % state)
utils.udevadm_settle()
out, err = utils.execute('parted', '-s', dev, 'set', str(num),
flag, state, check_exit_code=[0, 1])
LOG.debug('Parted output: \n%s' % out)
reread_partitions(dev, out=out)
def get_installed_packages(chroot):
"""The packages installed in chroot along with their versions"""
out, err = utils.execute('chroot', chroot, 'dpkg-query', '-W',
'-f="${Package} ${Version};;"')
pkglist = filter(None, out.split(';;'))
return dict([pkgver.split() for pkgver in pkglist])
def _mount_target(self, mount_dir, os_id=None, pseudo=True,
treat_mtab=True):
LOG.debug('Mounting target file systems: %s', mount_dir)
# Here we are going to mount all file systems in partition schema.
for fs in self.driver.partition_scheme.fs_sorted_by_depth(os_id):
if fs.mount == 'swap':
continue
mount = os.path.join(mount_dir, fs.mount.strip(os.sep))
utils.makedirs_if_not_exists(mount)
fu.mount_fs(fs.type, str(fs.device), mount)
if pseudo:
for path in ('/sys', '/dev', '/proc'):
utils.makedirs_if_not_exists(
os.path.join(mount_dir, path.strip(os.sep)))
fu.mount_bind(mount_dir, path)
if treat_mtab:
mtab = utils.execute('chroot', mount_dir, 'grep', '-v', 'rootfs',
'/proc/mounts')[0]
mtab_path = os.path.join(mount_dir, 'etc/mtab')
if os.path.islink(mtab_path):
os.remove(mtab_path)
with open(mtab_path, 'wt', encoding='utf-8') as f:
f.write(six.text_type(mtab))
def do_copyimage(self, os_id):
os_path = '/tmp/target/'
with self.mount_target(os_path, os_id, pseudo=False, treat_mtab=False):
for image in self.driver.image_scheme.get_os_images(os_id):
target_image_path = os.path.join(
os_path, image.target_device.strip(os.sep))
LOG.debug('Starting rsync from %s to %s', image.uri,
target_image_path)
rsync_flags = image.deployment_flags.get(
'rsync_flags', '-a -A -X')
utils.execute('rsync',
rsync_flags,
image.uri,
target_image_path,
check_exit_code=[0])
def get_lvm_config_value(chroot, section, name):
"""Get option value from current lvm configuration.
If option is not present in lvm.conf, None returns
"""
raw_value = utils.execute('chroot', chroot, 'lvm dumpconfig',
'/'.join((section, name)),
check_exit_code=[0, 5])[0]
if '=' not in raw_value:
return
raw_value = raw_value.split('=')[1].strip()
re_str = '"[^"]*"'
re_float = '\\d*\\.\\d*'
re_int = '\\d+'
tokens = re.findall('|'.join((re_str, re_float, re_int)), raw_value)
values = []
for token in tokens:
if re.match(re_str, token):
values.append(token.strip('"'))
elif re.match(re_float, token):
values.append(float(token))
elif re.match(re_int, token):
values.append(int(token))
if not values:
return
elif len(values) == 1:
return values[0]
else:
return values
def rsync_inject(src, dst):
"""Recursively copy the src to dst using full source paths
Example: suppose the source directory looks like
src/etc/myconfig
src/usr/bin/myscript
rsync_inject('src', '/tmp/chroot')
copies src/etc/myconfig to /tmp/chroot/etc/myconfig,
and src/usr/bin/myscript to /tmp/chroot/usr/bin/myscript,
respectively
"""
utils.makedirs_if_not_exists(os.path.dirname(dst))
LOG.debug('Rsync files from %s to: %s', src, dst)
utils.execute('rsync', '-rlptDKv', src + '/', dst + '/', logged=True)
def set_gpt_type(dev, num, type_guid):
"""Sets guid on a partition.
:param dev: A device file, e.g. /dev/sda.
:param num: Partition number
:param type_guid: Partition type guid. Must be one of those listed
on this page http://en.wikipedia.org/wiki/GUID_Partition_Table.
This method does not check whether type_guid is valid or not.
:returns: None
"""
# TODO(kozhukalov): check whether type_guid is valid
LOG.debug('Setting partition GUID: dev=%s num=%s guid=%s' %
(dev, num, type_guid))
utils.udevadm_settle()
utils.execute('sgdisk', '--typecode=%s:%s' % (num, type_guid),
dev, check_exit_code=[0])
def set_gpt_type(dev, num, type_guid):
"""Sets guid on a partition.
:param dev: A device file, e.g. /dev/sda.
:param num: Partition number
:param type_guid: Partition type guid. Must be one of those listed
on this page http://en.wikipedia.org/wiki/GUID_Partition_Table.
This method does not check whether type_guid is valid or not.
:returns: None
"""
# TODO(kozhukalov): check whether type_guid is valid
LOG.debug('Setting partition GUID: dev=%s num=%s guid=%s' %
(dev, num, type_guid))
utils.udevadm_settle()
utils.execute('sgdisk', '--typecode=%s:%s' % (num, type_guid),
dev, check_exit_code=[0])
def grub2_cfg_bundled(kernel_params='',
chroot='',
grub_timeout=10,
lvm_boot=False):
# NOTE(oberezovskyi): symlink is required because of grub2-probe fails
# to find device with root partition of fuel agent.
# It's actuall in the ram and "device" is "rootfs"
os.symlink(chroot, '/tmp/rootfs')
with grub2_prepare(kernel_params, chroot, grub_timeout, lvm_boot):
# NOTE(oberezovskyi): required to prevent adding boot entries for
# ramdisk
os.remove('/etc/grub.d/10_linux')
cmd = [guess_grub2_mkconfig(), '-o', chroot + '/boot/grub2/grub.cfg']
utils.execute(*cmd, run_as_root=True, cwd='/tmp/')
os.remove('/tmp/rootfs')
def remove_partition(dev, num):
LOG.debug('Trying to remove partition: dev=%s num=%s' % (dev, num))
if not any(x['fstype'] != 'free' and x['num'] == num
for x in info(dev)['parts']):
raise errors.PartitionNotFoundError('Partition %s not found' % num)
utils.udevadm_settle()
out, err = utils.execute('parted', '-s', dev, 'rm',
str(num), check_exit_code=[0, 1])
reread_partitions(dev, out=out)
def remove_partition(dev, num):
LOG.debug('Trying to remove partition: dev=%s num=%s' % (dev, num))
if not any(x['fstype'] != 'free' and x['num'] == num
for x in info(dev)['parts']):
raise errors.PartitionNotFoundError('Partition %s not found' % num)
utils.udevadm_settle()
out, err = utils.execute('parted', '-s', dev, 'rm',
str(num), check_exit_code=[0, 1])
reread_partitions(dev, out=out)
def guess_grub_version(chroot=''):
grub_install = guess_grub_install(chroot=chroot)
LOG.debug('Trying to run %s --version' % grub_install)
cmd = [grub_install, '--version']
if chroot:
cmd[:0] = ['chroot', chroot]
result = utils.execute(*cmd)
version = 1 if result[0].find('0.97') > 0 else 2
LOG.debug('Looks like grub version is %s' % version)
return version