def test_ceph_volume_command_2(): spec = DriveGroupSpec(host_pattern='*', data_devices=DeviceSelection(size='200GB:350GB', rotational=True), db_devices=DeviceSelection(size='200GB:350GB', rotational=False), wal_devices=DeviceSelection(size='10G') ) inventory = _mk_inventory(_mk_device(rotational=True)*2 + _mk_device(rotational=False)*2 + _mk_device(size="10.0 GB", rotational=False)*2 ) sel = drive_selection.DriveSelection(spec, inventory) cmd = translate.to_ceph_volume(spec, sel).run() assert cmd == ('lvm batch --no-auto /dev/sda /dev/sdb ' '--db-devices /dev/sdc /dev/sdd --wal-devices /dev/sde /dev/sdf ' '--yes --no-systemd')
def _daemon_add_osd(self, svc_arg=None): # type: (Optional[str]) -> HandleCommandResult """Create one or more OSDs""" usage = """ Usage: ceph orch daemon add osd host:device1,device2,... """ if not svc_arg: return HandleCommandResult(-errno.EINVAL, stderr=usage) try: host_name, block_device = svc_arg.split(":") block_devices = block_device.split(',') devs = DeviceSelection(paths=block_devices) drive_group = DriveGroupSpec( placement=PlacementSpec(host_pattern=host_name), data_devices=devs) except (TypeError, KeyError, ValueError): msg = "Invalid host:device spec: '{}'".format(svc_arg) + usage return HandleCommandResult(-errno.EINVAL, stderr=msg) completion = self.create_osds(drive_group) self._orchestrator_wait([completion]) raise_if_exception(completion) return HandleCommandResult(stdout=completion.result_str())
def test_osd_create_with_drive_groups(self, instance): # without orchestrator service fake_client = mock.Mock() instance.return_value = fake_client # Valid DriveGroup data = { 'method': 'drive_group', 'data': { 'service_type': 'osd', 'service_id': 'all_hdd', 'data_devices': { 'rotational': True }, 'host_pattern': '*' }, 'tracking_id': 'all_hdd, b_ssd' } # Without orchestrator service fake_client.available.return_value = False self._task_post('/api/osd', data) self.assertStatus(503) # With orchestrator service fake_client.available.return_value = True self._task_post('/api/osd', data) self.assertStatus(201) fake_client.osds.create.assert_called_with( DriveGroupSpec(placement=PlacementSpec(host_pattern='*'), service_id='all_hdd', service_type='osd', data_devices=DeviceSelection(rotational=True)))
def test_create_osds(self, _send_command, _get_connection, _save_host, _rm_host, cephadm_module): with self._with_host(cephadm_module, 'test'): dg = DriveGroupSpec('test', data_devices=DeviceSelection(paths=[''])) c = cephadm_module.create_osds([dg]) assert wait(cephadm_module, c) == ["Created osd(s) on host 'test'"]
def test_ceph_volume_command_6(): spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'), service_id='foobar', data_devices=DeviceSelection(rotational=False), journal_devices=DeviceSelection(rotational=True), journal_size='500M', objectstore='filestore') with pytest.raises(DriveGroupValidationError): spec.validate() inventory = _mk_inventory( _mk_device(rotational=True) * 2 + _mk_device(rotational=False) * 2) sel = drive_selection.DriveSelection(spec, inventory) cmd = translate.to_ceph_volume(sel, []).run() assert cmd == ('lvm batch --no-auto /dev/sdc /dev/sdd ' '--journal-size 500M --journal-devices /dev/sda /dev/sdb ' '--filestore --yes --no-systemd')
def test_drivegroup_pattern(): dg = DriveGroupSpec(PlacementSpec(host_pattern='node[1-3]'), service_id='foobar', data_devices=DeviceSelection(all=True)) assert dg.placement.filter_matching_hostspecs([ HostSpec('node{}'.format(i)) for i in range(10) ]) == ['node1', 'node2', 'node3']
def _create_osd(self, svc_arg=None, inbuf=None): # type: (str, str) -> HandleCommandResult """Create one or more OSDs""" usage = """ Usage: ceph orchestrator osd create -i <json_file> ceph orchestrator osd create host:device1,device2,... """ if inbuf: try: drive_group = DriveGroupSpec.from_json(json.loads(inbuf)) except ValueError as e: msg = 'Failed to read JSON input: {}'.format(str(e)) + usage return HandleCommandResult(-errno.EINVAL, stderr=msg) elif svc_arg: try: node_name, block_device = svc_arg.split(":") block_devices = block_device.split(',') except (TypeError, KeyError, ValueError): msg = "Invalid host:device spec: '{}'".format(svc_arg) + usage return HandleCommandResult(-errno.EINVAL, stderr=msg) devs = DeviceSelection(paths=block_devices) drive_group = DriveGroupSpec(node_name, data_devices=devs) else: return HandleCommandResult(-errno.EINVAL, stderr=usage) completion = self.create_osds(drive_group) self._orchestrator_wait([completion]) orchestrator.raise_if_exception(completion) self.log.warning(str(completion.result)) return HandleCommandResult(stdout=completion.result_str())
def test_apply_osd_save(self, _run_cephadm, cephadm_module: CephadmOrchestrator): _run_cephadm.return_value = ('{}', '', 0) with with_host(cephadm_module, 'test'): spec = DriveGroupSpec( service_id='foo', placement=PlacementSpec( host_pattern='*', ), data_devices=DeviceSelection( all=True ) ) c = cephadm_module.apply([spec]) assert wait(cephadm_module, c) == ['Scheduled osd.foo update...'] inventory = Devices([ Device( '/dev/sdb', available=True ), ]) cephadm_module.cache.update_host_devices_networks('test', inventory.devices, {}) _run_cephadm.return_value = (['{}'], '', 0) assert cephadm_module._apply_all_services() == False _run_cephadm.assert_any_call( 'test', 'osd', 'ceph-volume', ['--config-json', '-', '--', 'lvm', 'prepare', '--bluestore', '--data', '/dev/sdb', '--no-systemd'], env_vars=['CEPH_VOLUME_OSDSPEC_AFFINITY=foo'], error_ok=True, stdin='{"config": "", "keyring": ""}') _run_cephadm.assert_called_with('test', 'osd', 'ceph-volume', ['--', 'lvm', 'list', '--format', 'json'])
def _apply_osd(self, all_available_devices=False, inbuf=None): # type: (bool, Optional[str]) -> HandleCommandResult """Apply DriveGroupSpecs to create OSDs""" usage = """ Usage: ceph orch apply osd -i <json_file/yaml_file> ceph orch apply osd --use-all-devices """ if not inbuf and not all_available_devices: return HandleCommandResult(-errno.EINVAL, stderr=usage) if inbuf: if all_available_devices: raise OrchestratorError( '--all-available-devices cannot be combined with an osd spec' ) try: drivegroups = yaml.load_all(inbuf) dg_specs = [ServiceSpec.from_json(dg) for dg in drivegroups] except ValueError as e: msg = 'Failed to read JSON/YAML input: {}'.format( str(e)) + usage return HandleCommandResult(-errno.EINVAL, stderr=msg) else: dg_specs = [ DriveGroupSpec( service_id='all-available-devices', placement=PlacementSpec(host_pattern='*'), data_devices=DeviceSelection(all=True), ) ] completion = self.apply_drivegroups(dg_specs) self._orchestrator_wait([completion]) raise_if_exception(completion) return HandleCommandResult(stdout=completion.result_str())
def test_create_osds(self, cephadm_module): with with_host(cephadm_module, 'test'): dg = DriveGroupSpec(placement=PlacementSpec(host_pattern='test'), data_devices=DeviceSelection(paths=[''])) c = cephadm_module.create_osds(dg) out = wait(cephadm_module, c) assert out == "Created no osd(s) on host test; already created?"
def test_osd_create_with_drive_groups(self, instance): # without orchestrator service fake_client = mock.Mock() instance.return_value = fake_client # Valid DriveGroup data = self._get_drive_group_data() # Without orchestrator service fake_client.available.return_value = False self._task_post('/api/osd', data) self.assertStatus(503) # With orchestrator service fake_client.available.return_value = True fake_client.get_missing_features.return_value = [] self._task_post('/api/osd', data) self.assertStatus(201) dg_specs = [ DriveGroupSpec(placement=PlacementSpec(host_pattern='*'), service_id='all_hdd', service_type='osd', data_devices=DeviceSelection(rotational=True)) ] fake_client.osds.create.assert_called_with(dg_specs)
def test_ceph_volume_command_0(): spec = DriveGroupSpec(host_pattern='*', data_devices=DeviceSelection(all=True)) inventory = _mk_inventory(_mk_device() * 2) sel = drive_selection.DriveSelection(spec, inventory) cmd = translate.to_ceph_volume(spec, sel).run() assert cmd == 'lvm batch --no-auto /dev/sda /dev/sdb --yes --no-systemd'
def test_osd_create_with_drive_groups(self, instance): # without orchestrator service fake_client = mock.Mock() instance.return_value = fake_client # Valid DriveGroups data = { 'method': 'drive_groups', 'data': { 'all_hdd': { 'host_pattern': '*', 'data_devices': { 'rotational': True } }, 'b_ssd': { 'host_pattern': 'b', 'data_devices': { 'rotational': False } } }, 'tracking_id': 'all_hdd, b_ssd' } # Without orchestrator service fake_client.available.return_value = False self._task_post('/api/osd', data) self.assertStatus(503) # With orchestrator service fake_client.available.return_value = True self._task_post('/api/osd', data) self.assertStatus(201) fake_client.osds.create.assert_called_with([ DriveGroupSpec(host_pattern='*', name='all_hdd', data_devices=DeviceSelection(rotational=True)), DriveGroupSpec(host_pattern='b', name='b_ssd', data_devices=DeviceSelection(rotational=False)) ]) # Invalid DriveGroups data['data']['b'] = {'host_pattern1': 'aa'} self._task_post('/api/osd', data) self.assertStatus(400)
def test_create_osds(self, cephadm_module): with self._with_host(cephadm_module, 'test'): dg = DriveGroupSpec('test', data_devices=DeviceSelection(paths=[''])) c = cephadm_module.create_osds([dg]) assert wait(cephadm_module, c) == [ "Created no osd(s) on host test; already created?" ]
def test_ceph_volume_command_7(): spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'), data_devices=DeviceSelection(all=True), osd_id_claims={'host1': ['0', '1']}) inventory = _mk_inventory(_mk_device(rotational=True) * 2) sel = drive_selection.DriveSelection(spec, inventory) cmd = translate.to_ceph_volume(spec, sel, ['0', '1']).run() assert cmd == 'lvm batch --no-auto /dev/sda /dev/sdb --osd-ids 0 1 --yes --no-systemd'
def test_ceph_volume_command_5(): spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'), data_devices=DeviceSelection(rotational=True), objectstore='filestore') inventory = _mk_inventory(_mk_device(rotational=True) * 2) sel = drive_selection.DriveSelection(spec, inventory) cmd = translate.to_ceph_volume(spec, sel, []).run() assert cmd == 'lvm batch --no-auto /dev/sda /dev/sdb --filestore --yes --no-systemd'
def test_data_devices_prop(self, test_fix): test_fix = test_fix() assert test_fix.data_devices == DeviceSelection( model='42-RGB', size='30G:50G', vendor='samsung', limit=0, )
def test_driveselection_to_ceph_volume(self, cephadm_module, devices, preview, exp_command): with with_host(cephadm_module, 'test'): dg = DriveGroupSpec(service_id='test.spec', placement=PlacementSpec( host_pattern='test'), data_devices=DeviceSelection(paths=devices)) ds = DriveSelection(dg, Devices([Device(path) for path in devices])) preview = preview out = cephadm_module.osd_service.driveselection_to_ceph_volume(ds, [], preview) assert out in exp_command
def test_drivegroup_pattern(): dg = DriveGroupSpec(PlacementSpec(host_pattern='node[1-3]'), data_devices=DeviceSelection(all=True)) assert dg.placement.filter_matching_hosts( lambda label=None, as_hostspec=None: ['node{}'.format(i) for i in range(10)]) == [ 'node1', 'node2', 'node3' ]
def test_raw_ceph_volume_command_1(): spec = DriveGroupSpec( placement=PlacementSpec(host_pattern='*'), service_id='foobar', data_devices=DeviceSelection(rotational=True), db_devices=DeviceSelection(rotational=False), method='raw', ) spec.validate() inventory = _mk_inventory( _mk_device(rotational=True) + # data _mk_device(rotational=True) + # data _mk_device(rotational=False) # db ) sel = drive_selection.DriveSelection(spec, inventory) with pytest.raises(ValueError): cmds = translate.to_ceph_volume(sel, []).run()
def test_prepare_drivegroup(self, cephadm_module): with with_host(cephadm_module, 'test'): dg = DriveGroupSpec(placement=PlacementSpec(host_pattern='test'), data_devices=DeviceSelection(paths=[''])) out = cephadm_module.osd_service.prepare_drivegroup(dg) assert len(out) == 1 f1 = out[0] assert f1[0] == 'test' assert isinstance(f1[1], DriveSelection)
def test_ceph_volume_command_2(): spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'), service_id='foobar', data_devices=DeviceSelection(size='200GB:350GB', rotational=True), db_devices=DeviceSelection(size='200GB:350GB', rotational=False), wal_devices=DeviceSelection(size='10G')) spec.validate() inventory = _mk_inventory( _mk_device(rotational=True, size="300.00 GB") * 2 + _mk_device(rotational=False, size="300.00 GB") * 2 + _mk_device(size="10.0 GB", rotational=False) * 2) sel = drive_selection.DriveSelection(spec, inventory) cmds = translate.to_ceph_volume(sel, []).run() assert all(cmd == ( 'lvm batch --no-auto /dev/sda /dev/sdb ' '--db-devices /dev/sdc /dev/sdd --wal-devices /dev/sde /dev/sdf ' '--yes --no-systemd') for cmd in cmds), f'Expected {cmd} in {cmds}'
def test_ceph_volume_command_9(): spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'), service_id='foobar', data_devices=DeviceSelection(all=True), data_allocate_fraction=0.8) spec.validate() inventory = _mk_inventory(_mk_device() * 2) sel = drive_selection.DriveSelection(spec, inventory) cmd = translate.to_ceph_volume(sel, []).run() assert cmd == 'lvm batch --no-auto /dev/sda /dev/sdb --data-allocate-fraction 0.8 --yes --no-systemd'
def test_ceph_volume_command_8(): spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'), data_devices=DeviceSelection(rotational=True, model='INTEL SSDS'), db_devices=DeviceSelection(model='INTEL SSDP'), filter_logic='OR', osd_id_claims={}) inventory = _mk_inventory( _mk_device(rotational=True, size='1.82 TB', model='ST2000DM001-1ER1') + # data _mk_device(rotational=False, size="223.0 GB", model='INTEL SSDSC2KG24') + # data _mk_device(rotational=False, size="349.0 GB", model='INTEL SSDPED1K375GA') # wal/db ) sel = drive_selection.DriveSelection(spec, inventory) cmd = translate.to_ceph_volume(sel, []).run() assert cmd == 'lvm batch --no-auto /dev/sda /dev/sdb --db-devices /dev/sdc --yes --no-systemd'
def test_ceph_volume_command_0(): spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'), service_id='foobar', data_devices=DeviceSelection(all=True)) spec.validate() inventory = _mk_inventory(_mk_device() * 2) sel = drive_selection.DriveSelection(spec, inventory) cmds = translate.to_ceph_volume(sel, []).run() assert all( cmd == 'lvm batch --no-auto /dev/sda /dev/sdb --yes --no-systemd' for cmd in cmds), f'Expected {cmd} in {cmds}'
def test_ceph_volume_command_4(): spec = DriveGroupSpec(host_pattern='*', data_devices=DeviceSelection(size='200GB:350GB', rotational=True), db_devices=DeviceSelection(size='200GB:350GB', rotational=False), wal_devices=DeviceSelection(size='10G'), block_db_size='500M', block_wal_size='500M', osds_per_device=3, encrypted=True ) inventory = _mk_inventory(_mk_device(rotational=True)*2 + _mk_device(rotational=False)*2 + _mk_device(size="10.0 GB", rotational=False)*2 ) sel = drive_selection.DriveSelection(spec, inventory) cmd = translate.to_ceph_volume(spec, sel).run() assert cmd == ('lvm batch --no-auto /dev/sda /dev/sdb ' '--db-devices /dev/sdc /dev/sdd --wal-devices /dev/sde /dev/sdf ' '--block-wal-size 500M --block-db-size 500M --dmcrypt ' '--osds-per-device 3 --yes --no-systemd')
def test_preview_drivegroups_str(self, _run_c_v_command, _ds_to_cv, _prepare_dg, _find_store, cephadm_module): with self._with_host(cephadm_module, 'test'): dg = DriveGroupSpec(placement=PlacementSpec(host_pattern='test'), data_devices=DeviceSelection(paths=[''])) _find_store.return_value = [dg] _prepare_dg.return_value = [('host1', 'ds_dummy')] _run_c_v_command.return_value = ("{}", '', 0) cephadm_module.preview_drivegroups(drive_group_name='foo') _find_store.assert_called_once_with(service_name='foo') _prepare_dg.assert_called_once_with(dg) _run_c_v_command.assert_called_once()
def test_raw_ceph_volume_command_0(): spec = DriveGroupSpec( placement=PlacementSpec(host_pattern='*'), service_id='foobar', data_devices=DeviceSelection(rotational=True), db_devices=DeviceSelection(rotational=False), method='raw', ) spec.validate() inventory = _mk_inventory( _mk_device(rotational=True) + # data _mk_device(rotational=True) + # data _mk_device(rotational=False) + # db _mk_device(rotational=False) # db ) exp_cmds = [ 'raw prepare --bluestore --data /dev/sda --block.db /dev/sdc', 'raw prepare --bluestore --data /dev/sdb --block.db /dev/sdd' ] sel = drive_selection.DriveSelection(spec, inventory) cmds = translate.to_ceph_volume(sel, []).run() assert all(cmd in exp_cmds for cmd in cmds), f'Expected {exp_cmds} to match {cmds}'
def test_ceph_volume_command_5(): spec = DriveGroupSpec(placement=PlacementSpec(host_pattern='*'), service_id='foobar', data_devices=DeviceSelection(rotational=True), objectstore='filestore') with pytest.raises(DriveGroupValidationError): spec.validate() inventory = _mk_inventory(_mk_device(rotational=True) * 2) sel = drive_selection.DriveSelection(spec, inventory) cmds = translate.to_ceph_volume(sel, []).run() assert all( cmd == 'lvm batch --no-auto /dev/sda /dev/sdb --filestore --yes --no-systemd' for cmd in cmds), f'Expected {cmd} in {cmds}'
class TestDriveSelection(object): testdata = [ (DriveGroupSpec(placement=PlacementSpec(host_pattern='*'), service_id='foobar', data_devices=DeviceSelection(all=True)), _mk_inventory(_mk_device() * 5), ['/dev/sda', '/dev/sdb', '/dev/sdc', '/dev/sdd', '/dev/sde'], []), (DriveGroupSpec(placement=PlacementSpec(host_pattern='*'), service_id='foobar', data_devices=DeviceSelection(all=True, limit=3), db_devices=DeviceSelection(all=True)), _mk_inventory(_mk_device() * 5), ['/dev/sda', '/dev/sdb', '/dev/sdc'], ['/dev/sdd', '/dev/sde']), (DriveGroupSpec(placement=PlacementSpec(host_pattern='*'), service_id='foobar', data_devices=DeviceSelection(rotational=True), db_devices=DeviceSelection(rotational=False)), _mk_inventory( _mk_device(rotational=False) + _mk_device(rotational=True)), ['/dev/sdb'], ['/dev/sda']), (DriveGroupSpec(placement=PlacementSpec(host_pattern='*'), service_id='foobar', data_devices=DeviceSelection(rotational=True), db_devices=DeviceSelection(rotational=False)), _mk_inventory( _mk_device(rotational=True) * 2 + _mk_device(rotational=False)), ['/dev/sda', '/dev/sdb'], ['/dev/sdc']), (DriveGroupSpec(placement=PlacementSpec(host_pattern='*'), service_id='foobar', data_devices=DeviceSelection(rotational=True), db_devices=DeviceSelection(rotational=False)), _mk_inventory(_mk_device(rotational=True) * 2), ['/dev/sda', '/dev/sdb'], []), ] @pytest.mark.parametrize("spec,inventory,expected_data,expected_db", testdata) def test_disk_selection(self, spec, inventory, expected_data, expected_db): sel = drive_selection.DriveSelection(spec, inventory) assert [d.path for d in sel.data_devices()] == expected_data assert [d.path for d in sel.db_devices()] == expected_db