Example #1
0
def lvcreate(vgname, lvname, size):
    vg = filter(lambda x: x['name'] == vgname, vgdisplay())

    # check if vg exists
    if not vg:
        raise errors.VGNotFoundError(
            'Error while creating vg: vg %s not found' % vgname)
    # check if enough space is available
    if vg[0]['free'] < size:
        raise errors.NotEnoughSpaceError(
            'Error while creating lv: vg %s has only %s m of free space, '
            'but at least %s m is needed' % (vgname, vg[0]['free'], size))
    # check if lv already exists
    if filter(lambda x: x['name'] == lvname and x['vg'] == vgname,
              lvdisplay()):
        raise errors.LVAlreadyExistsError(
            'Error while creating lv: lv %s already exists' % lvname)
    # NOTE(agordeev): by default, lvcreate is configured to wipe signature
    # on allocated volume. '--yes' should be passed to avoid waiting for
    # user's confirmation:
    # "WARNING: <signature> signature detected on <device>. Wipe it? [y/n]"
    utils.execute('lvcreate',
                  '--yes',
                  '-L',
                  '%sm' % size,
                  '-n',
                  lvname,
                  vgname,
                  check_exit_code=[0])
Example #2
0
def mdremove(mdname):
    # check if md exists
    if mdname not in get_mdnames():
        raise errors.MDNotFoundError(
            'Error while removing md: md %s not found' % mdname)
    utils.execute('mdadm', '--stop', mdname, check_exit_code=[0])
    utils.execute('mdadm', '--remove', mdname, check_exit_code=[0, 1])
Example #3
0
def make_partition(dev, begin, end, ptype):
    LOG.debug('Trying to create a partition: dev=%s begin=%s end=%s' %
              (dev, begin, end))
    if ptype not in ('primary', 'logical'):
        raise errors.WrongPartitionSchemeError('Wrong partition type: %s' %
                                               ptype)

    # check begin >= end
    if begin >= end:
        raise errors.WrongPartitionSchemeError(
            'Wrong boundaries: begin >= end')

    # check if begin and end are inside one of free spaces available
    if not any(
            x['fstype'] == 'free' and begin >= x['begin'] and end <= x['end']
            for x in info(dev)['parts']):
        raise errors.WrongPartitionSchemeError(
            'Invalid boundaries: begin and end '
            'are not inside available free space')

    utils.execute('udevadm', 'settle', '--quiet', check_exit_code=[0])
    out, err = utils.execute('parted',
                             '-a',
                             'optimal',
                             '-s',
                             dev,
                             'unit',
                             'MiB',
                             'mkpart',
                             ptype,
                             str(begin),
                             str(end),
                             check_exit_code=[0, 1])
    LOG.debug('Parted output: \n%s' % out)
    reread_partitions(dev, out=out)
Example #4
0
    def do_configdrive(self):
        cc_output_path = os.path.join(CONF.tmp_path, 'cloud_config.txt')
        bh_output_path = os.path.join(CONF.tmp_path, 'boothook.txt')
        # NOTE:file should be strictly named as 'user-data'
        #      the same is for meta-data as well
        ud_output_path = os.path.join(CONF.tmp_path, 'user-data')
        md_output_path = os.path.join(CONF.tmp_path, 'meta-data')

        tmpl_dir = CONF.nc_template_path
        utils.render_and_save(
            tmpl_dir, self.configdrive_scheme.template_names('cloud_config'),
            self.configdrive_scheme.template_data(), cc_output_path)
        utils.render_and_save(
            tmpl_dir, self.configdrive_scheme.template_names('boothook'),
            self.configdrive_scheme.template_data(), bh_output_path)
        utils.render_and_save(
            tmpl_dir, self.configdrive_scheme.template_names('meta-data'),
            self.configdrive_scheme.template_data(), md_output_path)

        utils.execute('write-mime-multipart', '--output=%s' % ud_output_path,
                      '%s:text/cloud-boothook' % bh_output_path,
                      '%s:text/cloud-config' % cc_output_path)
        utils.execute('genisoimage', '-output', CONF.config_drive_path,
                      '-volid', 'cidata', '-joliet', '-rock', ud_output_path,
                      md_output_path)

        configdrive_device = self.partition_scheme.configdrive_device()
        if configdrive_device is None:
            raise errors.WrongPartitionSchemeError(
                'Error while trying to get configdrive device: '
                'configdrive device not found')
        self.image_scheme.add_image(uri='file://%s' % CONF.config_drive_path,
                                    target_device=configdrive_device,
                                    image_format='iso9660',
                                    container='raw')
Example #5
0
def lvcreate(vgname, lvname, size):
    vg = get_first_by_key_value(vgdisplay(), 'name', vgname)

    # check if vg exists
    if not vg:
        raise errors.VGNotFoundError(
            'Error while creating vg: vg %s not found' % vgname)
    # check if enough space is available
    if vg['free'] < size:
        raise errors.NotEnoughSpaceError(
            'Error while creating lv: vg %s has only %s m of free space, '
            'but at least %s m is needed' % (vgname, vg['free'], size))
    # check if lv already exists
    if next(
        (x for x in lvdisplay() if x['name'] == lvname and x['vg'] == vgname),
            False):
        raise errors.LVAlreadyExistsError(
            'Error while creating lv: lv %s already exists' % lvname)
    # NOTE(agordeev): by default, lvcreate is configured to wipe signature
    # on allocated volume. '--yes' should be passed to avoid waiting for
    # user's confirmation:
    # "WARNING: <signature> signature detected on <device>. Wipe it? [y/n]"
    # FIXME: the version of lvm2 shipped with Ubuntu 14.04 does not support
    # --yes option. fuel-agent should properly decomission the storage
    # (Ubuntu installer does that just fine).
    stdout, stderr = utils.execute('lvcreate', '--help')
    force_opt = '--yes' if '--yes' in stdout else ''
    cmd = 'lvcreate {force_opt} -L {size}m -n {lvname} {vgname}'.format(
        size=size, lvname=lvname, vgname=vgname, force_opt=force_opt)
    utils.execute(*cmd.split(), check_exit_code=[0])
Example #6
0
def set_partition_flag(dev, num, flag, state='on'):
    """Sets flag on a partition

    :param dev: A device file, e.g. /dev/sda.
    :param num: Partition number
    :param flag: Flag name. Must be one of 'bios_grub', 'legacy_boot',
    'boot', 'raid', 'lvm'
    :param state: Desiable flag state. 'on' or 'off'. Default is 'on'.

    :returns: None
    """
    # parted supports more flags but we are interested in
    # setting only this subset of them.
    # not all of these flags are compatible with one another.
    if flag not in ('bios_grub', 'legacy_boot', 'boot', 'raid', 'lvm'):
        raise errors.WrongPartitionSchemeError(
            'Unsupported partition flag: %s' % flag)
    if state not in ('on', 'off'):
        raise errors.WrongPartitionSchemeError(
            'Wrong partition flag state: %s' % state)
    utils.execute('parted',
                  '-s',
                  dev,
                  'set',
                  str(num),
                  flag,
                  state,
                  check_exit_code=[0])
Example #7
0
    def do_configdrive(self):
        LOG.debug('--- Creating configdrive (do_configdrive) ---')
        if CONF.prepare_configdrive:
            cc_output_path = os.path.join(CONF.tmp_path, 'cloud_config.txt')
            bh_output_path = os.path.join(CONF.tmp_path, 'boothook.txt')
            # NOTE:file should be strictly named as 'user-data'
            #      the same is for meta-data as well
            ud_output_path = os.path.join(CONF.tmp_path, 'user-data')
            md_output_path = os.path.join(CONF.tmp_path, 'meta-data')

            tmpl_dir = CONF.nc_template_path
            utils.render_and_save(
                tmpl_dir,
                self.driver.configdrive_scheme.template_names('cloud_config'),
                self.driver.configdrive_scheme.template_data(), cc_output_path)
            utils.render_and_save(
                tmpl_dir,
                self.driver.configdrive_scheme.template_names('boothook'),
                self.driver.configdrive_scheme.template_data(), bh_output_path)
            utils.render_and_save(
                tmpl_dir,
                self.driver.configdrive_scheme.template_names('meta_data'),
                self.driver.configdrive_scheme.template_data(), md_output_path)

            utils.execute('write-mime-multipart',
                          '--output=%s' % ud_output_path,
                          '%s:text/cloud-boothook' % bh_output_path,
                          '%s:text/cloud-config' % cc_output_path)
            utils.execute('genisoimage', '-output', CONF.config_drive_path,
                          '-volid', 'cidata', '-joliet', '-rock',
                          ud_output_path, md_output_path)

        if CONF.prepare_configdrive or os.path.isfile(CONF.config_drive_path):
            self._add_configdrive_image()
Example #8
0
def recompress_initramfs(chroot, compress='xz', initrd_mask='initrd*'):
    """Remove old and rebuild initrd

    :param chroot:
    :param compress: compression type for initrd
    :return:
    :initrd_mask: search kernel file by Unix style pathname
    """
    env_vars = copy.deepcopy(os.environ)
    add_env_vars = {'TMPDIR': '/tmp',
                    'TMP': '/tmp'}

    LOG.debug('Changing initramfs compression type to: %s', compress)
    utils.execute(
        'sed', '-i', 's/^COMPRESS=.*/COMPRESS={0}/'.format(compress),
        os.path.join(chroot, 'etc/initramfs-tools/initramfs.conf'))

    boot_dir = os.path.join(chroot, 'boot')
    initrds = glob.glob(os.path.join(boot_dir, initrd_mask))
    LOG.debug('Removing initrd images: %s', initrds)
    remove_files('/', initrds)

    env_vars.update(add_env_vars)
    LOG.info('Building initramfs')
    cmds = ['chroot', chroot, 'update-initramfs -v -c -k all']
    utils.execute(*cmds,
                  env_variables=env_vars, logged=True)
    LOG.debug('Running "update-initramfs" completed')
Example #9
0
def _activate_flavor(flavor=None):
    """Switch between cobbler distro profiles, in case dockerized system

    Unfortunately, we don't support switching between profiles "on fly",
    so to perform this we need:
    1) Update asute.yaml - which used by puppet to determine options
    2) Re-run puppet for cobbler(to perform default system update, regarding
       new profile)
    3) Re-run puppet for astute

    :param flavor: Switch between cobbler profile
    :return:
    """
    flavor = flavor.lower()
    if flavor not in consts.DISTROS:
        raise errors.WrongCobblerProfile(
            'Wrong cobbler profile passed: {0} \n '
            'possible profiles: {1}'.format(flavor,
                                            list(consts.DISTROS.keys())))
    _update_astute_yaml(consts.DISTROS[flavor]['astute_flavor'])
    _run_puppet(consts.COBBLER_MANIFEST)
    _run_puppet(consts.ASTUTE_MANIFEST)
    # restart astuted to be sure that it catches new profile
    LOG.debug('Reloading astuted')
    utils.execute('service', 'astute', 'restart')
Example #10
0
def set_partition_flag(dev, num, flag, state='on'):
    """Sets flag on a partition

    :param dev: A device file, e.g. /dev/sda.
    :param num: Partition number
    :param flag: Flag name. Must be one of 'bios_grub', 'legacy_boot',
    'boot', 'raid', 'lvm'
    :param state: Desiable flag state. 'on' or 'off'. Default is 'on'.

    :returns: None
    """
    LOG.debug('Trying to set partition flag: dev=%s num=%s flag=%s state=%s' %
              (dev, num, flag, state))
    # parted supports more flags but we are interested in
    # setting only this subset of them.
    # not all of these flags are compatible with one another.
    if flag not in ('bios_grub', 'legacy_boot', 'boot', 'raid', 'lvm'):
        raise errors.WrongPartitionSchemeError(
            'Unsupported partition flag: %s' % flag)
    if state not in ('on', 'off'):
        raise errors.WrongPartitionSchemeError(
            'Wrong partition flag state: %s' % state)
    utils.execute('udevadm', 'settle', '--quiet', check_exit_code=[0])
    out, err = utils.execute('parted', '-s', dev, 'set', str(num),
                             flag, state, check_exit_code=[0, 1])
    LOG.debug('Parted output: \n%s' % out)
    reread_partitions(dev, out=out)
Example #11
0
def grub2_cfg(kernel_params='', chroot='', grub_timeout=10):
    grub_defaults = chroot + guess_grub2_default(chroot=chroot)
    rekerparams = re.compile(r'^.*GRUB_CMDLINE_LINUX=.*')
    retimeout = re.compile(r'^.*GRUB_TIMEOUT=.*')
    rehidtimeout = re.compile(r'^.*GRUB_HIDDEN_TIMEOUT.*')
    new_content = ''
    with open(grub_defaults) as f:
        for line in f:
            line = rekerparams.sub(
                'GRUB_CMDLINE_LINUX="{kernel_params}"'.
                format(kernel_params=kernel_params), line)
            line = retimeout.sub('GRUB_TIMEOUT={grub_timeout}'.
                                 format(grub_timeout=grub_timeout), line)
            if not rehidtimeout.search(line):
                new_content += line
    # NOTE(agordeev): explicitly add record fail timeout, in order to
    # prevent user confirmation appearing if unexpected reboot occured.
    new_content += '\nGRUB_RECORDFAIL_TIMEOUT={grub_timeout}\n'.\
                   format(grub_timeout=grub_timeout)
    with open(grub_defaults, 'wt', encoding='utf-8') as f:
        f.write(six.text_type(new_content))
    cmd = [guess_grub2_mkconfig(chroot), '-o', guess_grub2_conf(chroot)]
    if chroot:
        cmd[:0] = ['chroot', chroot]
    utils.execute(*cmd, run_as_root=True)
Example #12
0
def mdremove(mdname):
    # check if md exists
    if mdname not in get_mdnames():
        raise errors.MDNotFoundError(
            'Error while removing md: md %s not found' % mdname)
    # FIXME: The issue faced was quiet hard to reproduce and to figure out the
    #       root cause. For unknown reason already removed md device is
    #       unexpectedly returning back after a while from time to time making
    #       new md device creation to fail.
    #           Still the actual reason of its failure is unknown, but after a
    #       searching on a web a mention was found about a race in udev
    #       http://dev.bizo.com/2012/07/mdadm-device-or-resource-busy.html
    #       The article recommends to disable udev's queue entirely during md
    #       device manipulation which sounds rather unappropriate for our case.
    #       And the link to original post on mailing list suggests to execute
    #       `udevadm settle` before removing the md device.
    #       here -> http://permalink.gmane.org/gmane.linux.raid/34027
    #           So, what was done. `udevadm settle` calls were placed just
    #       before any of `mdadm` calls and the analizyng the logs was started.
    #       According to the manual `settle` is an option that "Watches the
    #       udev event queue, and exits if all current events are handled".
    #       That means it will wait for udev's finishing of processing the
    #       events. According to the logs noticeable delay had been recognized
    #       between `udevadm settle` and the next `mdadm` call.
    #           The delay was about 150-200ms or even bigger. It was appeared
    #       right before the `mdadm --stop` call. That just means that udev was
    #       too busy with events when we start to modifiy md devices hard.
    #           Thus `udevadm settle` is helping to avoid the later failure and
    #       to prevent strange behaviour of md device.
    utils.udevadm_settle()
    utils.execute('mdadm', '--stop', mdname, check_exit_code=[0])
    utils.execute('mdadm', '--remove', mdname, check_exit_code=[0, 1])
Example #13
0
def override_lvm_config_value(chroot, section, name, value, lvm_conf_file):
    """Override option in LVM configuration.

    If option is not valid, then errors.ProcessExecutionError will be raised
    and lvm configuration will remain unchanged
    """
    lvm_conf_file = os.path.join(chroot, lvm_conf_file.lstrip('/'))
    updated_config = _update_option_in_lvm_raw_config(
        section, name, value,
        utils.execute('chroot', chroot, 'lvm dumpconfig')[0])
    lvm_conf_file_bak = '{}.bak.{}'.format(lvm_conf_file,
                                           time.strftime("%Y_%m_%d_%H_%M_%S"))
    shutil.copy(lvm_conf_file, lvm_conf_file_bak)
    LOG.debug('Backup for origin LVM configuration file: {}'
              ''.format(lvm_conf_file_bak))
    with open(lvm_conf_file, mode='w') as lvm_conf:
        lvm_conf.write(updated_config)

    # NOTE(sslypushenko) Extra cycle of dump/save lvm.conf is required to be
    # sure that updated configuration is valid and to adjust it to general
    # lvm.conf formatting
    try:
        current_config = utils.execute('chroot', chroot, 'lvm dumpconfig')[0]
        with open(lvm_conf_file, mode='w') as lvm_conf:
            lvm_conf.write(current_config)
        LOG.info('LVM configuration {} updated. '
                 'Option {}/{} gets new value: {}'
                 ''.format(lvm_conf_file, section, name, value))
    except errors.ProcessExecutionError as exc:
        shutil.move(lvm_conf_file_bak, lvm_conf_file)
        LOG.debug('Option {}/{} can not be updated with value {}. '
                  'Configuration restored'.format(section, name, value))
        raise exc
Example #14
0
def make_partition(dev, begin, end, ptype):
    LOG.debug('Trying to create a partition: dev=%s begin=%s end=%s' %
              (dev, begin, end))
    if ptype not in ('primary', 'logical'):
        raise errors.WrongPartitionSchemeError(
            'Wrong partition type: %s' % ptype)

    # check begin >= end
    if begin >= end:
        raise errors.WrongPartitionSchemeError(
            'Wrong boundaries: begin >= end')

    # check if begin and end are inside one of free spaces available
    if not any(x['fstype'] == 'free' and begin >= x['begin'] and
               end <= x['end'] for x in info(dev)['parts']):
        raise errors.WrongPartitionSchemeError(
            'Invalid boundaries: begin and end '
            'are not inside available free space')

    utils.execute('udevadm', 'settle', '--quiet', check_exit_code=[0])
    out, err = utils.execute(
        'parted', '-a', 'optimal', '-s', dev, 'unit', 'MiB',
        'mkpart', ptype, str(begin), str(end), check_exit_code=[0, 1])
    LOG.debug('Parted output: \n%s' % out)
    reread_partitions(dev, out=out)
Example #15
0
 def test_check_exit_code_boolean(self):
     utils.execute('/usr/bin/env', 'false', check_exit_code=False)
     self.assertRaises(processutils.ProcessExecutionError,
                       utils.execute,
                       '/usr/bin/env',
                       'false',
                       check_exit_code=True)
Example #16
0
def mdcreate(mdname, level, devices, metadata='default'):
    mds = mddisplay()

    # check if md device already exists
    if filter(lambda x: x['name'] == mdname, mds):
        raise errors.MDAlreadyExistsError(
            'Error while creating md: md %s already exists' % mdname)

    # check if level argument is valid
    supported_levels = ('0', '1', 'raid0', 'raid1', 'stripe', 'mirror')
    if level not in supported_levels:
        raise errors.MDWrongSpecError(
            'Error while creating md device: '
            'level must be one of: %s' % ', '.join(supported_levels))

    # check if all necessary devices exist
    if not set(devices).issubset(
            set([bd['device'] for bd in hu.list_block_devices(disks=False)])):
        raise errors.MDNotFoundError(
            'Error while creating md: at least one of devices is not found')

    # check if devices are not parts of some md array
    if set(devices) & \
            set(itertools.chain(*[md.get('devices', []) for md in mds])):
        raise errors.MDDeviceDuplicationError(
            'Error while creating md: at least one of devices is '
            'already in belongs to some md')

    # FIXME: mdadm will ask user to continue creating if any device appears to
    #       be a part of raid array. Superblock zeroing helps to avoid that.
    map(mdclean, devices)
    utils.execute('mdadm', '--create', '--force', mdname, '-e', metadata,
                  '--level=%s' % level,
                  '--raid-devices=%s' % len(devices), *devices,
                  check_exit_code=[0])
Example #17
0
def make_partition(dev, begin, end, ptype):
    if ptype not in ('primary', 'logical'):
        raise errors.WrongPartitionSchemeError('Wrong partition type: %s' %
                                               ptype)

    # check begin >= end
    if begin >= end:
        raise errors.WrongPartitionSchemeError(
            'Wrong boundaries: begin >= end')

    # check if begin and end are inside one of free spaces available
    if not any(
            x['fstype'] == 'free' and begin >= x['begin'] and end <= x['end']
            for x in info(dev)['parts']):
        raise errors.WrongPartitionSchemeError(
            'Invalid boundaries: begin and end '
            'are not inside available free space')

    utils.execute('parted',
                  '-a',
                  'optimal',
                  '-s',
                  dev,
                  'unit',
                  'MiB',
                  'mkpart',
                  ptype,
                  str(begin),
                  str(end),
                  check_exit_code=[0])
Example #18
0
def grub2_cfg(kernel_params='', chroot='', grub_timeout=10):
    grub_defaults = chroot + guess_grub2_default(chroot=chroot)
    rekerparams = re.compile(r'^.*GRUB_CMDLINE_LINUX=.*')
    retimeout = re.compile(r'^.*GRUB_TIMEOUT=.*')
    rehidtimeout = re.compile(r'^.*GRUB_HIDDEN_TIMEOUT.*')
    new_content = ''
    with open(grub_defaults) as f:
        for line in f:
            line = rekerparams.sub(
                'GRUB_CMDLINE_LINUX="{kernel_params}"'.format(
                    kernel_params=kernel_params), line)
            line = retimeout.sub(
                'GRUB_TIMEOUT={grub_timeout}'.format(
                    grub_timeout=grub_timeout), line)
            if not rehidtimeout.search(line):
                new_content += line
    # NOTE(agordeev): explicitly add record fail timeout, in order to
    # prevent user confirmation appearing if unexpected reboot occured.
    new_content += '\nGRUB_RECORDFAIL_TIMEOUT={grub_timeout}\n'.\
                   format(grub_timeout=grub_timeout)
    with open(grub_defaults, 'wt', encoding='utf-8') as f:
        f.write(six.text_type(new_content))
    cmd = [guess_grub2_mkconfig(chroot), '-o', guess_grub2_conf(chroot)]
    if chroot:
        cmd[:0] = ['chroot', chroot]
    utils.execute(*cmd, run_as_root=True)
Example #19
0
 def test_execute_ok_on_third_attempts(self, mock_popen, mock_sleep):
     process = mock.Mock()
     mock_popen.side_effect = [OSError, ValueError, process]
     process.communicate.return_value = (None, None)
     process.returncode = 0
     utils.execute("/usr/bin/env", "false", attempts=3)
     self.assertEqual(2 * [mock.call(CONF.execute_retry_delay)], mock_sleep.call_args_list)
Example #20
0
def override_lvm_config_value(chroot, section, name, value, lvm_conf_file):
    """Override option in LVM configuration.

    If option is not valid, then errors.ProcessExecutionError will be raised
    and lvm configuration will remain unchanged
    """
    lvm_conf_file = os.path.join(chroot, lvm_conf_file.lstrip('/'))
    updated_config = _update_option_in_lvm_raw_config(
        section, name, value,
        utils.execute('chroot', chroot, 'lvm dumpconfig')[0])
    lvm_conf_file_bak = '{}.bak.{}'.format(lvm_conf_file,
                                           time.strftime("%Y_%m_%d_%H_%M_%S"))
    shutil.copy(lvm_conf_file, lvm_conf_file_bak)
    LOG.debug('Backup for origin LVM configuration file: {}'
              ''.format(lvm_conf_file_bak))
    with open(lvm_conf_file, mode='w') as lvm_conf:
        lvm_conf.write(updated_config)

    # NOTE(sslypushenko) Extra cycle of dump/save lvm.conf is required to be
    # sure that updated configuration is valid and to adjust it to general
    # lvm.conf formatting
    try:
        current_config = utils.execute('chroot', chroot, 'lvm dumpconfig')[0]
        with open(lvm_conf_file, mode='w') as lvm_conf:
            lvm_conf.write(current_config)
        LOG.info('LVM configuration {} updated. '
                 'Option {}/{} gets new value: {}'
                 ''.format(lvm_conf_file, section, name, value))
    except errors.ProcessExecutionError as exc:
        shutil.move(lvm_conf_file_bak, lvm_conf_file)
        LOG.debug('Option {}/{} can not be updated with value {}. '
                  'Configuration restored'.format(section, name, value))
        raise exc
Example #21
0
def mount_fs(fs_type, fs_dev, fs_mount):
    utils.execute('mount',
                  '-t',
                  fs_type,
                  fs_dev,
                  fs_mount,
                  check_exit_code=[0])
Example #22
0
def lvcreate(vgname, lvname, size):
    vg = get_first_by_key_value(vgdisplay(), 'name', vgname)

    # check if vg exists
    if not vg:
        raise errors.VGNotFoundError(
            'Error while creating vg: vg %s not found' % vgname)
    # check if enough space is available
    if vg['free'] < size:
        raise errors.NotEnoughSpaceError(
            'Error while creating lv: vg %s has only %s m of free space, '
            'but at least %s m is needed' % (vgname, vg['free'], size))
    # check if lv already exists
    if next(
        (x for x in lvdisplay() if x['name'] == lvname and x['vg'] == vgname),
        False
    ):
        raise errors.LVAlreadyExistsError(
            'Error while creating lv: lv %s already exists' % lvname)
    # NOTE(agordeev): by default, lvcreate is configured to wipe signature
    # on allocated volume. '--yes' should be passed to avoid waiting for
    # user's confirmation:
    # "WARNING: <signature> signature detected on <device>. Wipe it? [y/n]"
    # FIXME: the version of lvm2 shipped with Ubuntu 14.04 does not support
    # --yes option. fuel-agent should properly decomission the storage
    # (Ubuntu installer does that just fine).
    stdout, stderr = utils.execute('lvcreate', '--help')
    force_opt = '--yes' if '--yes' in stdout else ''
    cmd = 'lvcreate {force_opt} -L {size}m -n {lvname} {vgname}'.format(
        size=size, lvname=lvname, vgname=vgname, force_opt=force_opt)
    utils.execute(*cmd.split(), check_exit_code=[0])
Example #23
0
def recompress_initramfs(chroot, compress='xz', initrd_mask='initrd*'):
    """Remove old and rebuild initrd

    :param chroot:
    :param compress: compression type for initrd
    :return:
    :initrd_mask: search kernel file by Unix style pathname
    """
    env_vars = copy.deepcopy(os.environ)
    add_env_vars = {'TMPDIR': '/tmp', 'TMP': '/tmp'}

    LOG.debug('Changing initramfs compression type to: %s', compress)
    utils.execute('sed', '-i', 's/^COMPRESS=.*/COMPRESS={0}/'.format(compress),
                  os.path.join(chroot, 'etc/initramfs-tools/initramfs.conf'))

    boot_dir = os.path.join(chroot, 'boot')
    initrds = glob.glob(os.path.join(boot_dir, initrd_mask))
    LOG.debug('Removing initrd images: %s', initrds)
    remove_files('/', initrds)

    env_vars.update(add_env_vars)
    LOG.info('Building initramfs')
    cmds = ['chroot', chroot, 'update-initramfs -v -c -k all']
    utils.execute(*cmds, env_variables=env_vars, logged=True)
    LOG.debug('Running "update-initramfs" completed')
Example #24
0
def mdremove(mdname):
    # check if md exists
    if mdname not in get_mdnames():
        raise errors.MDNotFoundError(
            'Error while removing md: md %s not found' % mdname)
    # FIXME: The issue faced was quiet hard to reproduce and to figure out the
    #       root cause. For unknown reason already removed md device is
    #       unexpectedly returning back after a while from time to time making
    #       new md device creation to fail.
    #           Still the actual reason of its failure is unknown, but after a
    #       searching on a web a mention was found about a race in udev
    #       http://dev.bizo.com/2012/07/mdadm-device-or-resource-busy.html
    #       The article recommends to disable udev's queue entirely during md
    #       device manipulation which sounds rather unappropriate for our case.
    #       And the link to original post on mailing list suggests to execute
    #       `udevadm settle` before removing the md device.
    #       here -> http://permalink.gmane.org/gmane.linux.raid/34027
    #           So, what was done. `udevadm settle` calls were placed just
    #       before any of `mdadm` calls and the analizyng the logs was started.
    #       According to the manual `settle` is an option that "Watches the
    #       udev event queue, and exits if all current events are handled".
    #       That means it will wait for udev's finishing of processing the
    #       events. According to the logs noticeable delay had been recognized
    #       between `udevadm settle` and the next `mdadm` call.
    #           The delay was about 150-200ms or even bigger. It was appeared
    #       right before the `mdadm --stop` call. That just means that udev was
    #       too busy with events when we start to modifiy md devices hard.
    #           Thus `udevadm settle` is helping to avoid the later failure and
    #       to prevent strange behaviour of md device.
    utils.udevadm_settle()
    utils.execute('mdadm', '--stop', mdname, check_exit_code=[0])
    utils.execute('mdadm', '--remove', mdname, check_exit_code=[0, 1])
Example #25
0
def _activate_flavor(flavor=None):
    """Switch between cobbler distro profiles, in case dockerized system

    Unfortunately, we don't support switching between profiles "on fly",
    so to perform this we need:
    1) Update asute.yaml - which used by puppet to determine options
    2) Re-run puppet for cobbler(to perform default system update, regarding
       new profile)
    3) Re-run puppet for astute

    :param flavor: Switch between cobbler profile
    :return:
    """
    flavor = flavor.lower()
    if flavor not in consts.DISTROS:
        raise errors.WrongCobblerProfile(
            'Wrong cobbler profile passed: {0} \n '
            'possible profiles: {1}'.format(flavor,
                                            list(consts.DISTROS.keys())))
    _update_astute_yaml(consts.DISTROS[flavor]['astute_flavor'])
    _run_puppet(consts.COBBLER_MANIFEST)
    _run_puppet(consts.ASTUTE_MANIFEST)
    # restart astuted to be sure that it catches new profile
    LOG.debug('Reloading astuted')
    utils.execute('service', 'astute', 'restart')
Example #26
0
def mdclean(device):
    # we don't care if device actually exists or not
    utils.execute('mdadm',
                  '--zero-superblock',
                  '--force',
                  device,
                  check_exit_code=[0])
Example #27
0
def grub2_install(install_devices, chroot=''):
    grub_install = guess_grub_install(chroot=chroot)
    for install_device in install_devices:
        cmd = [grub_install, install_device]
        if chroot:
            cmd[:0] = ['chroot', chroot]
        utils.execute(*cmd, run_as_root=True, check_exit_code=[0])
Example #28
0
def vgextend(vgname, pvname, *args):
    # check if vg exists
    if not get_first_by_key_value(vgdisplay(), 'name', vgname, False):
        raise errors.VGNotFoundError(
            'Error while extending vg: vg %s not found' % vgname)
    pvnames = [pvname] + list(args)
    _vg_attach_validate(pvnames)
    utils.execute('vgextend', vgname, *pvnames, check_exit_code=[0])
Example #29
0
def vgcreate(vgname, pvname, *args):
    # check if vg already exists
    if get_first_by_key_value(vgdisplay(), 'name', vgname, False):
        raise errors.VGAlreadyExistsError(
            'Error while creating vg: vg %s already exists' % vgname)
    pvnames = [pvname] + list(args)
    _vg_attach_validate(pvnames)
    utils.execute('vgcreate', vgname, *pvnames, check_exit_code=[0])
Example #30
0
 def test_execute_ok_on_third_attempts(self, mock_popen, mock_sleep):
     process = mock.Mock()
     mock_popen.side_effect = [OSError, ValueError, process]
     process.communicate.return_value = (None, None)
     process.returncode = 0
     utils.execute('/usr/bin/env', 'false', attempts=3)
     self.assertEqual(2 * [mock.call(CONF.execute_retry_delay)],
                      mock_sleep.call_args_list)
Example #31
0
def vgextend(vgname, pvname, *args):
    # check if vg exists
    if not get_first_by_key_value(vgdisplay(), 'name', vgname, False):
        raise errors.VGNotFoundError(
            'Error while extending vg: vg %s not found' % vgname)
    pvnames = [pvname] + list(args)
    _vg_attach_validate(pvnames)
    utils.execute('vgextend', vgname, *pvnames, check_exit_code=[0])
Example #32
0
def do_post_inst(chroot):
    # NOTE(agordeev): set up password for root
    utils.execute('sed', '-i', 's%root:[\*,\!]%root:' + ROOT_PASSWORD + '%',
                  os.path.join(chroot, 'etc/shadow'))
    # NOTE(agordeev): remove custom policy-rc.d which is needed to disable
    # execution of post/pre-install package hooks and start of services
    remove_files(chroot, ['usr/sbin/policy-rc.d'])
    clean_apt_settings(chroot)
Example #33
0
def vgcreate(vgname, pvname, *args):
    # check if vg already exists
    if get_first_by_key_value(vgdisplay(), 'name', vgname, False):
        raise errors.VGAlreadyExistsError(
            'Error while creating vg: vg %s already exists' % vgname)
    pvnames = [pvname] + list(args)
    _vg_attach_validate(pvnames)
    utils.execute('vgcreate', vgname, *pvnames, check_exit_code=[0])
Example #34
0
def vgextend(vgname, pvname, *args):
    # check if vg exists
    if not filter(lambda x: x['name'] == vgname, vgdisplay()):
        raise errors.VGNotFoundError(
            'Error while extending vg: vg %s not found' % vgname)
    pvnames = [pvname] + list(args)
    _vg_attach_validate(pvnames)
    utils.execute('vgextend', vgname, *pvnames, check_exit_code=[0])
Example #35
0
def vgextend(vgname, pvname, *args):
    # check if vg exists
    if not filter(lambda x: x['name'] == vgname, vgdisplay()):
        raise errors.VGNotFoundError(
            'Error while extending vg: vg %s not found' % vgname)
    pvnames = [pvname] + list(args)
    _vg_attach_validate(pvnames)
    utils.execute('vgextend', vgname, *pvnames, check_exit_code=[0])
Example #36
0
def vgcreate(vgname, pvname, *args):
    # check if vg already exists
    if filter(lambda x: x['name'] == vgname, vgdisplay()):
        raise errors.VGAlreadyExistsError(
            'Error while creating vg: vg %s already exists' % vgname)
    pvnames = [pvname] + list(args)
    _vg_attach_validate(pvnames)
    utils.execute('vgcreate', vgname, *pvnames, check_exit_code=[0])
Example #37
0
def vgcreate(vgname, pvname, *args):
    # check if vg already exists
    if filter(lambda x: x['name'] == vgname, vgdisplay()):
        raise errors.VGAlreadyExistsError(
            'Error while creating vg: vg %s already exists' % vgname)
    pvnames = [pvname] + list(args)
    _vg_attach_validate(pvnames)
    utils.execute('vgcreate', vgname, *pvnames, check_exit_code=[0])
Example #38
0
def do_post_inst(chroot):
    # NOTE(agordeev): set up password for root
    utils.execute('sed', '-i',
                  's%root:[\*,\!]%root:' + ROOT_PASSWORD + '%',
                  os.path.join(chroot, 'etc/shadow'))
    # NOTE(agordeev): remove custom policy-rc.d which is needed to disable
    # execution of post/pre-install package hooks and start of services
    remove_files(chroot, ['usr/sbin/policy-rc.d'])
    clean_apt_settings(chroot)
Example #39
0
def copy_update_certs(certs, chroot):
    """Try to copy and update CA certificates in chroot"""
    for cert in certs:
        rsync_inject(cert, chroot)
    utils.execute('chroot',
                  chroot,
                  'update-ca-certificates',
                  check_exit_code=False,
                  logged=True)
Example #40
0
def pvcreate(pvname, metadatasize=64, metadatacopies=2):
    # check if pv already exists
    if get_first_by_key_value(pvdisplay(), 'name', pvname, False):
        raise errors.PVAlreadyExistsError(
            'Error while creating pv: pv %s already exists' % pvname)
    utils.execute('pvcreate',
                  '--metadatacopies', str(metadatacopies),
                  '--metadatasize', str(metadatasize) + 'm',
                  pvname, check_exit_code=[0])
Example #41
0
def shrink_sparse_file(filename):
    """Shrinks file to its size of actual data. Only ext fs are supported."""
    utils.execute('e2fsck', '-y', '-f', filename)
    utils.execute('resize2fs', '-M', filename)
    data = hu.parse_simple_kv('dumpe2fs', filename)
    block_count = int(data['block count'])
    block_size = int(data['block size'])
    with open(filename, 'rwb+') as f:
        f.truncate(block_count * block_size)
Example #42
0
def create_sparse_tmp_file(dir, suffix, size=8192):
    """Creates sparse file.

    Creates file which consumes disk space more efficiently when the file
    itself is mostly empty.
    """
    tf = tempfile.NamedTemporaryFile(dir=dir, suffix=suffix, delete=False)
    utils.execute('truncate', '-s', '%sM' % size, tf.name)
    return tf.name
Example #43
0
def umount_fs(fs_mount):
    try:
        LOG.debug('Trying to umount {0}'.format(fs_mount))
        utils.execute('umount', fs_mount, check_exit_code=[0])
    except errors.ProcessExecutionError as e:
        LOG.warning('Error while umounting {0} '
                    'exc={1}'.format(fs_mount, e.message))
        LOG.debug('Trying lazy umounting {0}'.format(fs_mount))
        utils.execute('umount', '-l', fs_mount, check_exit_code=[0])
Example #44
0
def remove_partition(dev, num):
    LOG.debug('Trying to remove partition: dev=%s num=%s' % (dev, num))
    if not any(x['fstype'] != 'free' and x['num'] == num
               for x in info(dev)['parts']):
        raise errors.PartitionNotFoundError('Partition %s not found' % num)
    utils.execute('udevadm', 'settle', '--quiet', check_exit_code=[0])
    out, err = utils.execute('parted', '-s', dev, 'rm',
                             str(num), check_exit_code=[0, 1])
    reread_partitions(dev, out=out)
Example #45
0
def run_mksquashfs(chroot, output_name=None, compression_algorithm='xz'):
    """Pack the target system as squashfs using mksquashfs

    :param chroot: chroot system, to be squashfs'd
    :param output_name: output file name, might be a relative
     or an absolute path

    The kernel squashfs driver has to match with the user space squasfs tools.
    Use the mksquashfs provided by the target distro to achieve this.
    (typically the distro maintainers are smart enough to ship the correct
    version of mksquashfs)
    Use mksquashfs installed in the target system

    1)Mount tmpfs under chroot/mnt
    2)run mksquashfs inside a chroot
    3)move result files to dstdir
    """
    if not output_name:
        output_name = 'root.squashfs' + six.text_type(uuid.uuid4())
    utils.makedirs_if_not_exists(os.path.dirname(output_name))
    dstdir = os.path.dirname(output_name)
    temp = '.mksquashfs.tmp.' + six.text_type(uuid.uuid4())
    s_dst = os.path.join(chroot, 'mnt/dst')
    s_src = os.path.join(chroot, 'mnt/src')
    try:
        fu.mount_fs(
            'tmpfs', 'mnt_{0}'.format(temp),
            (os.path.join(chroot, 'mnt')),
            'rw,nodev,nosuid,noatime,mode=0755,size=4M')
        utils.makedirs_if_not_exists(s_src)
        utils.makedirs_if_not_exists(s_dst)
        # Bind mount the chroot to avoid including various temporary/virtual
        # files (/proc, /sys, /dev, and so on) into the image
        fu.mount_fs(None, chroot, s_src, opts='bind')
        fu.mount_fs(None, None, s_src, 'remount,bind,ro')
        fu.mount_fs(None, dstdir, s_dst, opts='bind')
        # run mksquashfs
        chroot_squash = os.path.join('/mnt/dst/' + temp)
        long_squash = os.path.join(chroot, 'mnt/dst/{0}'.format(temp))
        LOG.info('Building squashfs')
        utils.execute(
            'chroot', chroot, 'mksquashfs', '/mnt/src',
            chroot_squash,
            '-comp', compression_algorithm,
            '-no-progress', '-noappend', logged=True)
        # move to result name
        LOG.debug('Moving file: %s to: %s', long_squash, output_name)
        shutil.move(long_squash, output_name)
    except Exception as exc:
        LOG.error('squashfs_image build failed: %s', exc)
        raise
    finally:
        LOG.info('squashfs_image clean-up')
        stop_chrooted_processes(chroot, signal=signal.SIGTERM)
        fu.umount_fs(os.path.join(chroot, 'mnt/dst'))
        fu.umount_fs(os.path.join(chroot, 'mnt/src'))
        fu.umount_fs(os.path.join(chroot, 'mnt'))
Example #46
0
def shrink_sparse_file(filename):
    """Shrinks file to its size of actual data. Only ext fs are supported."""
    utils.execute('e2fsck', '-y', '-f', filename)
    utils.execute('resize2fs', '-M', filename)
    data = hu.parse_simple_kv('dumpe2fs', filename)
    block_count = int(data['block count'])
    block_size = int(data['block size'])
    with open(filename, 'rwb+') as f:
        f.truncate(block_count * block_size)
Example #47
0
def create_sparse_tmp_file(dir, suffix, size=8192):
    """Creates sparse file.

    Creates file which consumes disk space more efficiently when the file
    itself is mostly empty.
    """
    tf = tempfile.NamedTemporaryFile(dir=dir, suffix=suffix, delete=False)
    utils.execute('truncate', '-s', '%sM' % size, tf.name)
    return tf.name
Example #48
0
def pvcreate(pvname, metadatasize=64, metadatacopies=2):
    # check if pv already exists
    if filter(lambda x: x['name'] == pvname, pvdisplay()):
        raise errors.PVAlreadyExistsError(
            'Error while creating pv: pv %s already exists' % pvname)
    utils.execute('pvcreate',
                  '--metadatacopies', str(metadatacopies),
                  '--metadatasize', str(metadatasize) + 'm',
                  pvname, check_exit_code=[0])
Example #49
0
def umount_fs(fs_mount):
    try:
        LOG.debug('Trying to umount {0}'.format(fs_mount))
        utils.execute('umount', fs_mount, check_exit_code=[0])
    except errors.ProcessExecutionError as e:
        LOG.warning('Error while umounting {0} '
                    'exc={1}'.format(fs_mount, e.message))
        LOG.debug('Trying lazy umounting {0}'.format(fs_mount))
        utils.execute('umount', '-l', fs_mount, check_exit_code=[0])
Example #50
0
def mdremove(mdname):
    mds = mddisplay()

    # check if md exists
    if not filter(lambda x: x['name'] == mdname, mds):
        raise errors.MDNotFoundError(
            'Error while removing md: md %s not found' % mdname)

    utils.execute('mdadm', '--stop', mdname, check_exit_code=[0])
    utils.execute('mdadm', '--remove', mdname, check_exit_code=[0, 1])
Example #51
0
def info(dev):
    utils.execute('udevadm', 'settle', '--quiet', check_exit_code=[0])
    output = utils.execute('parted', '-s', dev, '-m',
                           'unit', 'MiB',
                           'print', 'free',
                           check_exit_code=[0])[0]
    LOG.debug('Info output: \n%s' % output)
    result = parse_partition_info(output)
    LOG.debug('Info result: %s' % result)
    return result
Example #52
0
def _run_puppet(manifest=None):
    """Run puppet apply

    :param manifest:
    :return:
    """
    LOG.debug('Trying apply manifest: %s', manifest)
    utils.execute('puppet', 'apply', '--detailed-exitcodes',
                  '-dv', manifest, logged=True,
                  check_exit_code=[0, 2], attempts=2)
Example #53
0
def mdremove(mdname):
    mds = mddisplay()

    # check if md exists
    if not filter(lambda x: x['name'] == mdname, mds):
        raise errors.MDNotFoundError(
            'Error while removing md: md %s not found' % mdname)

    utils.execute('mdadm', '--stop', mdname, check_exit_code=[0])
    utils.execute('mdadm', '--remove', mdname, check_exit_code=[0, 1])
Example #54
0
def _run_puppet(manifest=None):
    """Run puppet apply

    :param manifest:
    :return:
    """
    LOG.debug('Trying apply manifest: %s', manifest)
    utils.execute('puppet', 'apply', '--detailed-exitcodes',
                  '-dv', manifest, logged=True,
                  check_exit_code=[0, 2], attempts=2)
Example #55
0
def deattach_loop(loop, check_exit_code=[0]):
    LOG.debug('Trying to figure out if loop device %s is attached', loop)
    output = utils.execute('losetup', '-a')[0]
    for line in output.split('\n'):
        # output lines are assumed to have the following format
        # /dev/loop0: [fd03]:130820 (/dev/loop0)
        if loop == line.split(':')[0]:
            LOG.debug('Loop device %s seems to be attached. '
                      'Trying to detach.', loop)
            utils.execute('losetup', '-d', loop,
                          check_exit_code=check_exit_code)
Example #56
0
def deattach_loop(loop, check_exit_code=[0]):
    LOG.debug('Trying to figure out if loop device %s is attached', loop)
    output = utils.execute('losetup', '-a')[0]
    for line in output.split('\n'):
        # output lines are assumed to have the following format
        # /dev/loop0: [fd03]:130820 (/dev/loop0)
        if loop == line.split(':')[0]:
            LOG.debug('Loop device %s seems to be attached. '
                      'Trying to detach.', loop)
            utils.execute('losetup', '-d', loop,
                          check_exit_code=check_exit_code)
Example #57
0
def pvremove(pvname):
    pv = filter(lambda x: x['name'] == pvname, pvdisplay())

    # check if pv exists
    if not pv:
        raise errors.PVNotFoundError(
            'Error while removing pv: pv %s not found' % pvname)
    # check if pv is attached to some vg
    if pv[0]['vg'] is not None:
        raise errors.PVBelongsToVGError('Error while removing pv: '
                                        'pv belongs to vg %s' % pv[0]['vg'])
    utils.execute('pvremove', '-ff', '-y', pvname, check_exit_code=[0])
Example #58
0
def pvremove(pvname):
    pv = get_first_by_key_value(pvdisplay(), 'name', pvname)

    # check if pv exists
    if not pv:
        raise errors.PVNotFoundError(
            'Error while removing pv: pv %s not found' % pvname)
    # check if pv is attached to some vg
    if pv['vg'] is not None:
        raise errors.PVBelongsToVGError('Error while removing pv: '
                                        'pv belongs to vg %s' % pv['vg'])
    utils.execute('pvremove', '-ff', '-y', pvname, check_exit_code=[0])
Example #59
0
def pvremove(pvname):
    pv = filter(lambda x: x['name'] == pvname, pvdisplay())

    # check if pv exists
    if not pv:
        raise errors.PVNotFoundError(
            'Error while removing pv: pv %s not found' % pvname)
    # check if pv is attached to some vg
    if pv[0]['vg'] is not None:
        raise errors.PVBelongsToVGError('Error while removing pv: '
                                        'pv belongs to vg %s' % pv[0]['vg'])
    utils.execute('pvremove', '-ff', '-y', pvname, check_exit_code=[0])