def _generate_sub_disks(self,
note: HaNoteStruct,
services: List[FidWithType],
cns: ConsulUtil,
kv_cache=None) -> List[HaNoteStruct]:
disk_list = []
new_state = note.no_state
proc_fid = Fid.from_struct(note.no_id)
state = (ObjHealth.OK if new_state == HaNoteStruct.M0_NC_ONLINE else
ObjHealth.OFFLINE)
is_mkfs = self._is_mkfs(proc_fid)
mkfs_down = is_mkfs and state != ObjHealth.OK
if not mkfs_down:
for svc in services:
disk_list += cns.get_disks_by_parent_process(proc_fid, svc.fid)
if disk_list:
# XXX: Need to check the current state of the device, transition
# to ONLINE only in case of an explicit request or iff the prior
# state of the device is UNKNOWN/OFFLINE.
if not mkfs_down:
# We don't mark the devices as failed if the process is MKFS
# and if its effective status is STOPPED (see EOS-24124).
cns.update_drive_state(disk_list, state, device_event=False)
LOG.debug('proc fid=%s encloses %d disks as follows: %s', proc_fid,
len(disk_list), disk_list)
drive_ha_notes: List[HaNoteStruct] = []
for drive_id in disk_list:
# Get the drive state from Consul KV.
dstate = cns.get_sdev_state(ObjT.DRIVE, drive_id.key)
drive_ha_notes.append(
HaNoteStruct(no_id=drive_id.to_c(), no_state=dstate))
return drive_ha_notes
def _generate_sub_disks(self, note: HaNoteStruct, services: List,
cns: ConsulUtil) -> List[HaNoteStruct]:
disk_list = []
new_state = note.no_state
proc_fid = Fid.from_struct(note.no_id)
for svc in services:
disk_list += cns.get_disks_by_parent_process(proc_fid, svc.fid)
LOG.debug('proc fid=%s encloses %d disks as follows: %s', proc_fid,
len(disk_list), disk_list)
return [
HaNoteStruct(no_id=x.to_c(), no_state=new_state) for x in disk_list
]
def _generate_sub_disks(self, note: HaNoteStruct, services: List,
cns: ConsulUtil) -> List[HaNoteStruct]:
disk_list = []
new_state = note.no_state
proc_fid = Fid.from_struct(note.no_id)
for svc in services:
disk_list += cns.get_disks_by_parent_process(proc_fid, svc.fid)
if disk_list:
state = (ServiceHealth.OK if new_state == HaNoteStruct.M0_NC_ONLINE
else ServiceHealth.OFFLINE)
# XXX: Need to check the current state of the device, transition
# to ONLINE only in case of an explicit request or iff the prior
# state of the device is UNKNOWN/OFFLINE.
cns.update_drive_state(disk_list, state, device_event=False)
LOG.debug('proc fid=%s encloses %d disks as follows: %s', proc_fid,
len(disk_list), disk_list)
drive_ha_notes: List[HaNoteStruct] = []
for drive_id in disk_list:
# Get the drive state from Consul KV.
dstate = cns.get_sdev_state(ObjT.DRIVE, drive_id.key)
drive_ha_notes.append(
HaNoteStruct(no_id=drive_id.to_c(), no_state=dstate))
return drive_ha_notes
def test_process_failure(self):
consul_util = ConsulUtil()
consul_cache = InvocationCache()
ffi = Mock(spec=['init_motr_api'])
motr = Motr(ffi, None, None, consul_util)
# Setup for the test: notification of a process failure
# - failure here is an ios service and a disk
# - dummy Consul reports all processes on the node are failed
# - expect the node, enclosure, controller, drive,
# process, and service to all be marked as failed
#
# Static names and fids for the setup are given here.
node_name = 'testnode'
hax_fid = Fid(0x7200000000000001, 0x6)
site_fid = Fid(0x5300000000000001, 0x1)
rack_fid = Fid(0x6100000000000001, 0x2)
node_fid = Fid(0x6e00000000000001, 0x3)
encl_fid = Fid(0x6500000000000001, 0x4)
ctrl_fid = Fid(0x6300000000000001, 0x5)
process_fid = Fid(0x7200000000000001, 0x15)
service_fid = Fid(0x7300000000000001, 0xe)
service_fid_typed = FidWithType(fid=service_fid, service_type='ios')
drive_fid = Fid(0x6b00000000000001, 0x11)
ctrl_path = 'm0conf/sites/{}/racks/{}/encls/{}/ctrls/{}'.format(
site_fid, rack_fid, encl_fid, ctrl_fid)
ctrl_state = '{"state": "M0_NC_FAILED"}'
# Set mock return values for the necessary Consul calls
motr._is_mkfs = Mock(return_value=False)
consul_util.get_hax_fid = Mock(return_value=hax_fid)
consul_util.is_proc_client = Mock(return_value=False)
consul_util.get_services_by_parent_process = Mock(
return_value=[service_fid_typed])
consul_util.get_disks_by_parent_process = Mock(
return_value=[drive_fid])
consul_util.get_process_node = Mock(return_value=node_name)
consul_util.get_node_name_by_fid = Mock(return_value=node_name)
consul_util.get_node_fid = Mock(return_value=node_fid)
consul_util.get_node_encl_fid = Mock(return_value=encl_fid)
consul_util.get_node_ctrl_fids = Mock(return_value=[ctrl_fid])
# These failure indications are here to trigger specific code paths for
# node failure. Additional tests can cover different scenarios (e.g.
# drive failure but node still up), which will set differernt results
# for these calls.
consul_util.all_io_services_failed = Mock(return_value=True)
consul_util.get_sdev_state = Mock(
return_value=HaNoteStruct.M0_NC_FAILED)
consul_util.get_ctrl_state = Mock(
return_value=m0HaObjState.M0_NC_FAILED)
consul_util.get_ctrl_state_updates = Mock(
return_value=[PutKV(key=ctrl_path, value=ctrl_state)])
# We'll use these mocks to check that expected updates are happening.
consul_util.update_drive_state = Mock()
consul_util.set_process_state = Mock()
consul_util.set_node_state = Mock()
consul_util.set_encl_state = Mock()
motr._ha_broadcast = Mock()
motr._write_updates = Mock()
# Send the mock event.
motr.broadcast_ha_states(
[HAState(fid=process_fid, status=ObjHealth.FAILED)],
notify_devices=True,
broadcast_hax_only=False,
kv_cache=consul_cache)
# ConsulUtil is responsible for the actual KV updates, just check
# here that the appropriate util function is called for each
# component.
consul_util.update_drive_state.assert_called_with([drive_fid],
ObjHealth.OFFLINE,
device_event=False)
consul_util.set_process_state.assert_called_with(
process_fid, ObjHealth.FAILED)
consul_util.set_node_state.assert_called_with(node_fid,
ObjHealth.FAILED)
consul_util.set_encl_state.assert_called_with(encl_fid,
ObjHealth.FAILED,
kv_cache=consul_cache)
# This KV update is batched, so the check looks different.
motr._write_updates.assert_any_call(
[PutKV(key=ctrl_path, value=ctrl_state)], consul_cache)
# Check hax broadcast. We should see states updated to FAILED.
broadcast_list = motr._ha_broadcast.call_args[0][0]
self.assertTrue(_has_failed_note(broadcast_list, node_fid))
self.assertTrue(_has_failed_note(broadcast_list, encl_fid))
self.assertTrue(_has_failed_note(broadcast_list, ctrl_fid))
self.assertTrue(_has_failed_note(broadcast_list, process_fid))
self.assertTrue(_has_failed_note(broadcast_list, service_fid))
self.assertTrue(_has_failed_note(broadcast_list, drive_fid))