def __init__(self, data): super(NailgunBuildImage, self).__init__(data) self._image_scheme = objects.ImageScheme() self._partition_scheme = objects.PartitionScheme() self.parse_schemes() self._operating_system = self.parse_operating_system()
def parse_partition_scheme(self): partition_scheme = objects.PartitionScheme() for obj in ('lv', 'pv', 'fs', 'vg', 'md', 'parted'): attr = '{0}s'.format(obj) parse_method = getattr(self, 'parse_{0}_data'.format(obj)) raw = self.partition_data.get(attr, {}) setattr(partition_scheme, attr, parse_method(raw)) return partition_scheme
def setUp(self): self.mxn = mixins.MountableMixin() self.mxn.driver = mock.MagicMock(spec=nailgun.Nailgun) self.mxn.driver.partition_scheme = objects.PartitionScheme()
def _get_hw_partition_schema(self): """Reads disks/partitions from underlying hardware. Does not rely on deploy_config """ # NOTE(lobur): Only disks/partitions currently supported. # No vgs/volumes LOG.debug('--- Reading HW partition scheme from the node ---') partition_schema = objects.PartitionScheme() disk_infos = [pu.info(disk['name']) for disk in self.hu_disks] fstab = self._find_hw_fstab() LOG.debug('Scanning all disks on the node') for disk_info in disk_infos: parted = partition_schema.add_parted( name=disk_info['generic']['dev'], label=disk_info['generic']['table'], install_bootloader=disk_info['generic']['has_bootloader']) LOG.debug('Scanning all partitions on disk %s ' % disk_info['generic']['dev']) for part in disk_info['parts']: if part.get('fstype', '') == 'free': LOG.debug('Skipping a free partition at:' 'begin=%s, end=%s' % (part.get('begin'), part.get('end'))) continue LOG.debug('Adding partition: ' 'name=%s size=%s to hw schema' % (part.get('disk_dev'), part.get('size'))) # NOTE(lobur): avoid use of parted.add_partition to omit # counting logic; use real data instead. partition = objects.Partition(name=part.get('name'), count=part.get('num'), device=part.get('disk_dev'), begin=part.get('begin'), end=part.get('end'), partition_type=part.get('type'), flags=part.get('flags')) parted.partitions.append(partition) mnt_point = self._get_mount_point_from_fstab( fstab, part['uuid']) if mnt_point: LOG.debug( 'Adding filesystem: ' 'device=%s fs_type=%s mount_point=%s ' 'to hw schema' % (part.get('name'), part.get('fstype'), mnt_point)) partition_schema.add_fs(device=part.get('name'), mount=mnt_point, fs_type=part.get('fstype', '')) else: LOG.warning("Not adding %s filesystem to hw_schema because" " it has no mount point in fstab" % part.get('name')) return partition_schema
def _get_partition_scheme(self): """Reads disk/partitions volumes/vgs from given deploy_config Translating different ids (name, path, scsi) to name via scanning/comparing the underlying node hardware. """ LOG.debug('--- Preparing partition scheme ---') # TODO(oberezovskyi): make validator work data = self._partition_data() ks_spaces_validator.validate(data, 'ironic') data = convert_size(data) partition_schema = objects.PartitionScheme() multiboot_installed = False LOG.debug('Looping over all disks in provision data') for disk in self._ks_disks: # # skipping disk if there are no volumes with size >0 # # to be allocated on it which are not boot partitions if all((v["size"] <= 0 for v in disk["volumes"] if v.get("mount") != "/boot")): continue LOG.debug('Processing disk type:%s id:%s' % (disk['id']['type'], disk['id']['value'])) LOG.debug('Adding gpt table on disk type:%s id:%s' % (disk['id']['type'], disk['id']['value'])) parted = partition_schema.add_parted(name=self._disk_dev(disk), label='gpt', disk_size=disk['size']) # TODO(lobur): do not add partitions implicitly, they may fail # partition verification parted.add_partition(size=DEFAULT_GRUB_SIZE, flags=['bios_grub']) if self.is_multiboot and not multiboot_installed: multiboot_installed = True multiboot_partition = parted.add_partition(size=100) partition_schema.add_fs(device=multiboot_partition.name, mount='multiboot', fs_type='ext4', fstab_enabled=False, os_id=[]) LOG.debug('Looping over all volumes on disk type:%s id:%s' % (disk['id']['type'], disk['id']['value'])) for volume in disk['volumes']: LOG.debug('Processing volume: ' 'name=%s type=%s size=%s mount=%s vg=%s ' 'keep_data=%s' % (volume.get('name'), volume.get('type'), volume.get('size'), volume.get('mount'), volume.get('vg'), volume.get('keep_data'))) if volume['size'] <= 0: LOG.debug('Volume size is zero. Skipping.') continue FUNC_MAP = { 'partition': self._process_partition, 'raid': self._process_raid, 'pv': self._process_pv } FUNC_MAP[volume['type']](volume, disk, parted, partition_schema) LOG.debug('Looping over all volume groups in provision data') for vg in self._ks_vgs: self._process_vg(vg, partition_schema) partition_schema.elevate_keep_data() return partition_schema
def parse_partition_scheme(self): LOG.debug('--- Preparing partition scheme ---') data = self.partition_data() ks_spaces_validator.validate(data) partition_scheme = objects.PartitionScheme() ceph_osds = self._num_ceph_osds() journals_left = ceph_osds ceph_journals = self._num_ceph_journals() LOG.debug('Looping over all disks in provision data') for disk in self.ks_disks: # skipping disk if there are no volumes with size >0 # to be allocated on it which are not boot partitions if all((v["size"] <= 0 for v in disk["volumes"] if v["type"] != "boot" and v.get("mount") != "/boot")): continue LOG.debug('Processing disk %s' % disk['name']) LOG.debug('Adding gpt table on disk %s' % disk['name']) parted = partition_scheme.add_parted(name=self._disk_dev(disk), label='gpt') if disk in self.boot_disks: # we install bootloader only on every suitable disk LOG.debug('Adding bootloader stage0 on disk %s' % disk['name']) parted.install_bootloader = True # legacy boot partition LOG.debug('Adding bios_grub partition on disk %s: size=24' % disk['name']) parted.add_partition(size=24, flags=['bios_grub']) # uefi partition (for future use) LOG.debug('Adding UEFI partition on disk %s: size=200' % disk['name']) parted.add_partition(size=200) LOG.debug('Looping over all volumes on disk %s' % disk['name']) for volume in disk['volumes']: LOG.debug('Processing volume: ' 'name=%s type=%s size=%s mount=%s vg=%s' % (volume.get('name'), volume.get('type'), volume.get('size'), volume.get('mount'), volume.get('vg'))) if volume['size'] <= 0: LOG.debug('Volume size is zero. Skipping.') continue if volume.get('name') == 'cephjournal': LOG.debug('Volume seems to be a CEPH journal volume. ' 'Special procedure is supposed to be applied.') # We need to allocate a journal partition for each ceph OSD # Determine the number of journal partitions we need on # each device ratio = int(math.ceil(float(ceph_osds) / ceph_journals)) # No more than 10GB will be allocated to a single journal # partition size = volume["size"] / ratio if size > 10240: size = 10240 # This will attempt to evenly spread partitions across # multiple devices e.g. 5 osds with 2 journal devices will # create 3 partitions on the first device and 2 on the # second if ratio < journals_left: end = ratio else: end = journals_left for i in range(0, end): journals_left -= 1 if volume['type'] == 'partition': LOG.debug('Adding CEPH journal partition on ' 'disk %s: size=%s' % (disk['name'], size)) prt = parted.add_partition(size=size) LOG.debug('Partition name: %s' % prt.name) if 'partition_guid' in volume: LOG.debug('Setting partition GUID: %s' % volume['partition_guid']) prt.set_guid(volume['partition_guid']) continue if volume['type'] in ('partition', 'pv', 'raid'): if volume.get('mount') != '/boot': LOG.debug('Adding partition on disk %s: size=%s' % (disk['name'], volume['size'])) prt = parted.add_partition(size=volume['size'], keep_data=volume.get( 'keep_data', False)) LOG.debug('Partition name: %s' % prt.name) elif volume.get('mount') == '/boot' \ and not self._boot_partition_done \ and disk in self.boot_disks: LOG.debug( 'Adding /boot partition on disk %s: ' 'size=%s', disk['name'], volume['size']) prt = parted.add_partition(size=volume['size'], keep_data=volume.get( 'keep_data', False)) LOG.debug('Partition name: %s', prt.name) self._boot_partition_done = True else: LOG.debug( 'No need to create partition on disk %s. ' 'Skipping.', disk['name']) continue if volume['type'] == 'partition': if 'partition_guid' in volume: LOG.debug('Setting partition GUID: %s' % volume['partition_guid']) prt.set_guid(volume['partition_guid']) if 'mount' in volume and volume['mount'] != 'none': LOG.debug('Adding file system on partition: ' 'mount=%s type=%s' % (volume['mount'], volume.get('file_system', 'xfs'))) partition_scheme.add_fs( device=prt.name, mount=volume['mount'], fs_type=volume.get('file_system', 'xfs'), fs_label=volume.get('disk_label')) if volume['mount'] == '/boot' and not self._boot_done: self._boot_done = True if volume['type'] == 'pv': LOG.debug('Creating pv on partition: pv=%s vg=%s' % (prt.name, volume['vg'])) lvm_meta_size = volume.get('lvm_meta_size', 64) # The reason for that is to make sure that # there will be enough space for creating logical volumes. # Default lvm extension size is 4M. Nailgun volume # manager does not care of it and if physical volume size # is 4M * N + 3M and lvm metadata size is 4M * L then only # 4M * (N-L) + 3M of space will be available for # creating logical extensions. So only 4M * (N-L) of space # will be available for logical volumes, while nailgun # volume manager might reguire 4M * (N-L) + 3M # logical volume. Besides, parted aligns partitions # according to its own algorithm and actual partition might # be a bit smaller than integer number of mebibytes. if lvm_meta_size < 10: raise errors.WrongPartitionSchemeError( 'Error while creating physical volume: ' 'lvm metadata size is too small') metadatasize = int(math.floor((lvm_meta_size - 8) / 2)) metadatacopies = 2 partition_scheme.vg_attach_by_name( pvname=prt.name, vgname=volume['vg'], metadatasize=metadatasize, metadatacopies=metadatacopies) if volume['type'] == 'raid': if 'mount' in volume and \ volume['mount'] not in ('none', '/boot'): LOG.debug('Attaching partition to RAID ' 'by its mount point %s' % volume['mount']) metadata = 'default' if self.have_grub1_by_default: metadata = '0.90' LOG.debug( 'Going to use MD metadata version {0}. ' 'The version was guessed at the data has ' 'been given about the operating system.'.format( metadata)) partition_scheme.md_attach_by_mount( device=prt.name, mount=volume['mount'], fs_type=volume.get('file_system', 'xfs'), fs_label=volume.get('disk_label'), metadata=metadata) if 'mount' in volume and volume['mount'] == '/boot' and \ not self._boot_done: LOG.debug('Adding file system on partition: ' 'mount=%s type=%s' % (volume['mount'], volume.get('file_system', 'ext2'))) partition_scheme.add_fs( device=prt.name, mount=volume['mount'], fs_type=volume.get('file_system', 'ext2'), fs_label=volume.get('disk_label')) self._boot_done = True # this partition will be used to put there configdrive image if (partition_scheme.configdrive_device() is None and CONF.prepare_configdrive or os.path.isfile(CONF.config_drive_path)): LOG.debug('Adding configdrive partition on disk %s: size=20' % disk['name']) parted.add_partition(size=20, configdrive=True) # checking if /boot is expected to be created if self._have_boot_partition(self.ks_disks) and \ (not self._boot_partition_done or not self._boot_done): raise errors.WrongPartitionSchemeError( '/boot partition has not been created for some reasons') LOG.debug('Looping over all volume groups in provision data') for vg in self.ks_vgs: LOG.debug('Processing vg %s' % vg['id']) LOG.debug('Looping over all logical volumes in vg %s' % vg['id']) for volume in vg['volumes']: LOG.debug('Processing lv %s' % volume['name']) if volume['size'] <= 0: LOG.debug('LogicalVolume size is zero. Skipping.') continue if volume['type'] == 'lv': LOG.debug('Adding lv to vg %s: name=%s, size=%s' % (vg['id'], volume['name'], volume['size'])) lv = partition_scheme.add_lv(name=volume['name'], vgname=vg['id'], size=volume['size']) if 'mount' in volume and volume['mount'] != 'none': LOG.debug('Adding file system on lv: ' 'mount=%s type=%s' % (volume['mount'], volume.get('file_system', 'xfs'))) partition_scheme.add_fs( device=lv.device_name, mount=volume['mount'], fs_type=volume.get('file_system', 'xfs'), fs_label=volume.get('disk_label')) partition_scheme.elevate_keep_data() return partition_scheme
def test_do_build_image(self, mock_umount_target, mock_mount_target, mock_yaml_dump, mock_open, mock_shutil_move, mock_os, mock_utils, mock_fu, mock_bu, mock_set_apt_repos): loops = [objects.Loop(), objects.Loop()] self.mgr.driver._image_scheme = objects.ImageScheme([ objects.Image('file:///fake/img.img.gz', loops[0], 'ext4', 'gzip'), objects.Image('file:///fake/img-boot.img.gz', loops[1], 'ext2', 'gzip') ]) self.mgr.driver._partition_scheme = objects.PartitionScheme() self.mgr.driver.partition_scheme.add_fs(device=loops[0], mount='/', fs_type='ext4') self.mgr.driver.partition_scheme.add_fs(device=loops[1], mount='/boot', fs_type='ext2') self.mgr.driver.metadata_uri = 'file:///fake/img.yaml' self.mgr.driver._operating_system = objects.Ubuntu( repos=[ objects.DEBRepo('ubuntu', 'http://fakeubuntu', 'trusty', 'fakesection', priority=900), objects.DEBRepo('ubuntu_zero', 'http://fakeubuntu_zero', 'trusty', 'fakesection', priority=None), objects.DEBRepo('mos', 'http://fakemos', 'mosX.Y', 'fakesection', priority=1000) ], packages=['fakepackage1', 'fakepackage2']) self.mgr.driver.operating_system.proxies = objects.RepoProxies( proxies={'fake': 'fake'}, direct_repo_addr_list='fake_addr') self.mgr.driver.operating_system.minor = 4 self.mgr.driver.operating_system.major = 14 mock_os.path.exists.return_value = False mock_os.path.join.return_value = '/tmp/imgdir/proc' mock_os.path.basename.side_effect = ['img.img.gz', 'img-boot.img.gz'] mock_bu.create_sparse_tmp_file.side_effect = \ ['/tmp/img', '/tmp/img-boot'] mock_bu.attach_file_to_free_loop_device.side_effect = [ '/dev/loop0', '/dev/loop1' ] mock_bu.mkdtemp_smart.return_value = '/tmp/imgdir' getsize_side = [20, 2, 10, 1] mock_os.path.getsize.side_effect = getsize_side md5_side = [ 'fakemd5_raw', 'fakemd5_gzip', 'fakemd5_raw_boot', 'fakemd5_gzip_boot' ] mock_utils.calculate_md5.side_effect = md5_side mock_bu.containerize.side_effect = ['/tmp/img.gz', '/tmp/img-boot.gz'] mock_bu.stop_chrooted_processes.side_effect = [ False, True, False, True ] metadata = { 'os': { 'name': 'Ubuntu', 'major': 14, 'minor': 4 }, 'packages': self.mgr.driver.operating_system.packages } self.mgr.do_build_image() self.assertEqual([ mock.call('/fake/img.img.gz'), mock.call('/fake/img-boot.img.gz') ], mock_os.path.exists.call_args_list) self.assertEqual([ mock.call(dir=CONF.image_build_dir, suffix=CONF.image_build_suffix, size=CONF.sparse_file_size) ] * 2, mock_bu.create_sparse_tmp_file.call_args_list) self.assertEqual([ mock.call('/tmp/img', loop_device_major_number=CONF.loop_device_major_number, max_loop_devices_count=CONF.max_loop_devices_count, max_attempts=CONF.max_allowed_attempts_attach_image), mock.call('/tmp/img-boot', loop_device_major_number=CONF.loop_device_major_number, max_loop_devices_count=CONF.max_loop_devices_count, max_attempts=CONF.max_allowed_attempts_attach_image) ], mock_bu.attach_file_to_free_loop_device.call_args_list) self.assertEqual([ mock.call( fs_type='ext4', fs_options='', fs_label='', dev='/dev/loop0'), mock.call( fs_type='ext2', fs_options='', fs_label='', dev='/dev/loop1') ], mock_fu.make_fs.call_args_list) mock_bu.mkdtemp_smart.assert_called_once_with(CONF.image_build_dir, CONF.image_build_suffix) mock_mount_target.assert_called_once_with('/tmp/imgdir', treat_mtab=False, pseudo=False) self.assertEqual([mock.call('/tmp/imgdir')] * 2, mock_bu.suppress_services_start.call_args_list) mock_bu.run_debootstrap.assert_called_once_with( uri='http://fakeubuntu', suite='trusty', chroot='/tmp/imgdir', attempts=CONF.fetch_packages_attempts, proxies={'fake': 'fake'}, direct_repo_addr='fake_addr') mock_bu.set_apt_get_env.assert_called_once_with() mock_bu.pre_apt_get.assert_called_once_with( '/tmp/imgdir', allow_unsigned_file=CONF.allow_unsigned_file, force_ipv4_file=CONF.force_ipv4_file, proxies={'fake': 'fake'}, direct_repo_addr='fake_addr') driver_os = self.mgr.driver.operating_system mock_set_apt_repos.assert_called_with( '/tmp/imgdir', driver_os.repos, proxies=driver_os.proxies.proxies, direct_repo_addrs=driver_os.proxies.direct_repo_addr_list) mock_utils.makedirs_if_not_exists.assert_called_once_with( '/tmp/imgdir/proc') self.assertEqual([ mock.call('tune2fs', '-O', '^has_journal', '/dev/loop0'), mock.call('tune2fs', '-O', 'has_journal', '/dev/loop0') ], mock_utils.execute.call_args_list) mock_fu.mount_bind.assert_called_once_with('/tmp/imgdir', '/proc') mock_bu.populate_basic_dev.assert_called_once_with('/tmp/imgdir') mock_bu.run_apt_get.assert_called_once_with( '/tmp/imgdir', packages=['fakepackage1', 'fakepackage2'], attempts=CONF.fetch_packages_attempts) mock_bu.do_post_inst.assert_called_once_with( '/tmp/imgdir', allow_unsigned_file=CONF.allow_unsigned_file, force_ipv4_file=CONF.force_ipv4_file) signal_calls = mock_bu.stop_chrooted_processes.call_args_list self.assertEqual( 2 * [ mock.call('/tmp/imgdir', signal=signal.SIGTERM), mock.call('/tmp/imgdir', signal=signal.SIGKILL) ], signal_calls) self.assertEqual([mock.call('/tmp/imgdir/proc')] * 2, mock_fu.umount_fs.call_args_list) self.assertEqual([mock.call('/tmp/imgdir', pseudo=False)] * 2, mock_umount_target.call_args_list) self.assertEqual([mock.call('/dev/loop0'), mock.call('/dev/loop1')] * 2, mock_bu.deattach_loop.call_args_list) self.assertEqual([mock.call('/tmp/img'), mock.call('/tmp/img-boot')], mock_bu.shrink_sparse_file.call_args_list) self.assertEqual([ mock.call('/tmp/img'), mock.call('/fake/img.img.gz'), mock.call('/tmp/img-boot'), mock.call('/fake/img-boot.img.gz') ], mock_os.path.getsize.call_args_list) self.assertEqual([ mock.call('/tmp/img', 20), mock.call('/fake/img.img.gz', 2), mock.call('/tmp/img-boot', 10), mock.call('/fake/img-boot.img.gz', 1) ], mock_utils.calculate_md5.call_args_list) self.assertEqual([ mock.call('/tmp/img', 'gzip', chunk_size=CONF.data_chunk_size), mock.call('/tmp/img-boot', 'gzip', chunk_size=CONF.data_chunk_size) ], mock_bu.containerize.call_args_list) mock_open.assert_called_once_with('/fake/img.yaml', 'wt', encoding='utf-8') self.assertEqual([ mock.call('/tmp/img.gz', '/fake/img.img.gz'), mock.call('/tmp/img-boot.gz', '/fake/img-boot.img.gz') ], mock_shutil_move.call_args_list) for repo in self.mgr.driver.operating_system.repos: metadata.setdefault('repos', []).append({ 'type': 'deb', 'name': repo.name, 'uri': repo.uri, 'suite': repo.suite, 'section': repo.section, 'priority': repo.priority, 'meta': repo.meta }) metadata['images'] = [{ 'raw_md5': md5_side[0], 'raw_size': getsize_side[0], 'raw_name': None, 'container_name': os.path.basename(self.mgr.driver.image_scheme.images[0].uri.split( 'file://', 1)[1]), 'container_md5': md5_side[1], 'container_size': getsize_side[1], 'container': self.mgr.driver.image_scheme.images[0].container, 'format': self.mgr.driver.image_scheme.images[0].format }, { 'raw_md5': md5_side[2], 'raw_size': getsize_side[2], 'raw_name': None, 'container_name': os.path.basename(self.mgr.driver.image_scheme.images[1].uri.split( 'file://', 1)[1]), 'container_md5': md5_side[3], 'container_size': getsize_side[3], 'container': self.mgr.driver.image_scheme.images[1].container, 'format': self.mgr.driver.image_scheme.images[1].format }] mock_yaml_dump.assert_called_once_with(metadata, stream=mock_open())
def setUp(self): super(TestPartitionScheme, self).setUp() self.p_scheme = objects.PartitionScheme()