def node_to_drive_fid(self, node_name: str, drive: str):
sdev_fid: Fid = Fid(0, 0)
# We extract the sdev fid as follows,
# e.g. node_name=ssc-vm-c-0553.colo.seagate.com
# drive=/dev/vdf
# 1. m0conf/nodes/ssc-vm-c-0553.colo.seagate.com/processes/41/
# services/ios:43
# 2. Create ioservice motr fid
# 3. fetch consul kv for ios fid,
# m0conf/nodes/0x6e00000000000001:0x20/processes/
# 0x7200000000000001:0x29/services/0x7300000000000001:0x2b/
# sdevs/0x6400000000000001:0x2c:
# {"path": "/dev/vdf", "state": "M0_NC_UNKNOWN"}
# 4. find drive name in the json value and extract sdev fid from the
# key 0x6400000000000001:0x2c
# 5. Create sdev fid from sdev fid key.
process_items = self.kv.kv_get(f'm0conf/nodes/{node_name}/processes',
recurse=True)
for x in process_items:
if '/ios' in x['Key']:
fidk_ios = x['Value']
ios_fid = create_service_fid(int(fidk_ios))
sdev_items = self.kv.kv_get('m0conf/nodes', recurse=True)
for x in sdev_items:
if f'/{ios_fid}/' in x['Key']:
if json.loads(x['Value'])['path'] == drive:
# Using constant index 8 for the sdev fid.
# Fix this by changing the Consul schema to have
# mapping of drive path to sdev direct mapping.
sdev_fid_item = x['Key'].split('/')[8]
sdev_fidk = Fid.parse(sdev_fid_item).key
sdev_fid = create_sdev_fid(sdev_fidk)
break
return self.sdev_to_drive_fid(sdev_fid)
def fn():
proc_state_to_objhealth = {
'M0_CONF_HA_PROCESS_STARTING': ObjHealth.OFFLINE,
'M0_CONF_HA_PROCESS_STARTED': ObjHealth.OK,
'M0_CONF_HA_PROCESS_STOPPING': ObjHealth.OFFLINE,
'M0_CONF_HA_PROCESS_STOPPED': ObjHealth.OFFLINE
}
# import pudb.remote
# pudb.remote.set_trace(term_size=(80, 40), port=9998)
ha_states: List[HAState] = []
LOG.debug('process status: %s', data)
for item in data:
proc_val = base64.b64decode(item['Value'])
proc_status = json.loads(str(proc_val.decode('utf-8')))
LOG.debug('process update item key %s item val: %s',
item['Key'].split('/')[1], proc_status)
proc_fid = Fid.parse(item['Key'].split('/')[1])
proc_state = proc_status['state']
proc_type = proc_status['type']
if (proc_type != 'M0_CONF_HA_PROCESS_M0MKFS'
and proc_state in ('M0_CONF_HA_PROCESS_STARTED',
'M0_CONF_HA_PROCESS_STOPPED')):
ha_states.append(
HAState(fid=proc_fid,
status=proc_state_to_objhealth[proc_state]))
planner.add_command(
BroadcastHAStates(states=ha_states, reply_to=None))
def get_disks_by_parent_process(self,
process_fid: Fid,
svc_fid: Fid) -> List[Fid]:
node_items = self.kv.kv_get('m0conf/nodes', recurse=True)
# This is the RegExp to match the keys in Consul KV that describe
# the Motr processes and services that are enclosed into the Motr
# process that has the given process_fid.
#
# Note: we assume that process_fid uniquely identifies the given
# process within the whole cluster (that's why we are not interested
# in the hostnames here).
#
# Examples of the key that will match:
# m0conf/nodes/0x6e00000000000001:0x3b/processes/
# 0x7200000000000001:0x44/services/0x7300000000000001:0x46
regex = re.compile(
f'^m0conf\\/.*\\/processes\\/{process_fid}\\/services\\/'
f'{svc_fid}\\/(.+)$')
disks = []
for node in node_items:
match_result = re.match(regex, node['Key'])
if not match_result:
continue
sdev_fid_item = node['Key'].split('/')[8]
sdev_fidk = Fid.parse(sdev_fid_item).key
sdev_fid = create_sdev_fid(sdev_fidk)
disk_fid = self.sdev_to_drive_fid(sdev_fid)
disks.append(disk_fid)
return disks
def get_node_encl_fid(self, node: str) -> Optional[Fid]:
"""
Returns the fid of the enclosure for the given node.
Parameters:
node : hostname of the node.
"""
# Example,
# {
# "key": "m0conf/sites/0x5300000000000001:0x1/
# racks/0x6100000000000001:0x2/encls/
# 0x6500000000000001:0x4",
# "value": "{\"node\": \"0x6e00000000000001:0x3\",
# \"state\": \"M0_NC_UNKNOWN\"}"
# },
node_fid = self.get_node_fid(node)
if not node_fid:
return None
encl_items = self.kv.kv_get('m0conf/sites', recurse=True)
regex = re.compile('^m0conf\\/.*\\/racks\\/.*\\/encls\\/([^/]+)$')
for encl in encl_items:
match_result = re.match(regex, encl['Key'])
if not match_result:
continue
encl_value = json.loads(encl['Value'])
if 'node' in encl_value and encl_value['node'] == str(node_fid):
encl_fid: str = match_result.group(1)
return Fid.parse(encl_fid)
return None
def get_node_ctrl_fid(self, node: str) -> Optional[Fid]:
"""
Returns the fid of the controller for the given node.
Parameters:
node : hostname of the node.
"""
# Example,
# {
# "key": "m0conf/sites/0x5300000000000001:0x1/
# racks/0x6100000000000001:0x2/encls/
# 0x6500000000000001:0x4/ctrls/0x6300000000000001:0x5",
# },
encl_fid = self.get_node_encl_fid(node)
if not encl_fid:
return None
ctrl_items = self.kv.kv_get('m0conf/sites', recurse=True)
regex = re.compile(
f'^m0conf\\/.*\\/racks\\/.*\\/encls\\/{encl_fid}\\/ctrls\\/'
'([^/]+)$')
for ctrl in ctrl_items:
match_result = re.match(regex, ctrl['Key'])
if not match_result:
continue
ctrl_fid: str = match_result.group(1)
return Fid.parse(ctrl_fid)
return None
def get_service_process_fid(self, svc_fid: Fid) -> Fid:
assert ObjT.SERVICE.value == svc_fid.container
node_items = self.kv.kv_get('m0conf/nodes', recurse=True)
keys = self.get_service_keys(node_items, svc_fid.key)
assert len(keys) == 1
process_fid: str = keys[0].split('/')[4]
pfid = Fid.parse(process_fid)
return pfid
def handle_ioq_stob_error(self, payload: Dict[str, Any]) -> None:
fid = Fid.parse(payload['conf_sdev'])
if fid.is_null():
LOG.debug('Fid is 0:0. Skipping the message.')
return
q: Queue = Queue(1)
self.planner.add_command(
BroadcastHAStates(states=[HAState(fid, status=ObjHealth.FAILED)],
reply_to=q))
ids: List[MessageId] = q.get()
self.herald.wait_for_any(HaLinkMessagePromise(ids))
def handle_ioq_stob_error(self, payload: Dict[str, Any]) -> None:
fid = Fid.parse(payload['conf_sdev'])
if fid.is_null():
logging.debug('Fid is 0:0. Skipping the message.')
return
q: Queue = Queue(1)
self.queue.put(
BroadcastHAStates(states=[HAState(fid, status='offline')],
reply_to=q))
ids: List[MessageId] = q.get()
self.herald.wait_for_any(HaLinkMessagePromise(ids))
def get_node_fid(self, node: str) -> Optional[Fid]:
"""
Returns the fid of the given node.
Parameters:
node : hostname of the node.
"""
# Example,
# m0conf/nodes/
# 0x6e00000000000001:0x3:{"name": "ssc-vm-1623.colo.seagate.com",
# "state": "M0_NC_UNKNOWN"}
node_items = self.kv.kv_get('m0conf/nodes', recurse=True)
for item in node_items:
item_value = json.loads(item['Value'])
if 'name' in item_value and item_value['name'] == node:
node_fid: str = str(item['Key'].split('/')[2])
return Fid.parse(node_fid)
return None
def drive_to_sdev_fid(self, drive_fid: Fid) -> Fid:
# We extract the sdev fid as follows,
# e.g. drive_fid=0x6400000000000001:0x2d
# 1. m0conf/sites/0x5300000000000001:0x1/racks/0x6100000000000001:0x2/
# encls/0x6500000000000001:0x21/ctrls/0x6300000000000001:0x22/
# drives/0x6b00000000000001:0x2d:{"sdev": "0x6400000000000001:0x2c",
# "state": "M0_NC_UNKNOWN"}
# 2. Fetch Consul kv for sdev fid
# 3. Extract sdev fid key from the sdev fid.
# 4. Create sdev fid from fid key.
sdev_fid: Fid = Fid(0, 0)
sdev_items = self.kv.kv_get('m0conf/sites', recurse=True)
regex = re.compile(f'^m0conf\\/.*\\/drives/{drive_fid}$')
for x in sdev_items:
match_result = re.match(regex, x['Key'])
if not match_result:
continue
sdev_fid_item = json.loads(x['Value'])['sdev']
sdev_fidk = Fid.parse(sdev_fid_item).key
sdev_fid = create_sdev_fid(sdev_fidk)
break
return sdev_fid
def sdev_to_drive_fid(self, sdev_fid: Fid):
# We extract the drive fid as follows,
# e.g. sdev_fid=0x6400000000000001:0x2c
# 1. m0conf/sites/0x5300000000000001:0x1/racks/0x6100000000000001:0x2/
# encls/0x6500000000000001:0x21/ctrls/0x6300000000000001:0x22/
# drives/0x6b00000000000001:0x2d:{"sdev": "0x6400000000000001:0x2c",
# "state": "M0_NC_UNKNOWN"}
# 2. Fetch Consul kv for drive fid
# 3. Extract drive fid key from the drive fid.
# 4. Create drive fid from fid key.
drive_fid: Fid = Fid(0, 0)
drive_items = self.kv.kv_get('m0conf/sites', recurse=True)
for x in drive_items:
if '/drives/' in x['Key']:
if json.loads(x['Value'])['sdev'] == f'{sdev_fid}':
# Using constant index 10 for the drive fid.
# Fix this by changing the Consul schema to have
# mapping of sdev fid to drive fid direct mapping.
drive_fid_item = x['Key'].split('/')[10]
drive_fidk = Fid.parse(drive_fid_item).key
drive_fid = create_drive_fid(drive_fidk)
break
return drive_fid
def _get_pver_with_pver_status(
self, motr: Motr) -> Optional[Dict[str, PverInfo]]:
'''
Storing a map of pver_fid and its state.
Ex of pver state:
PverInfo(fid=0x7600000000000001:0x3e, state=0,
data_units=1, parity_units=0, pool_width=10, unit_size=0)
Pver data is stored in consul kv in format
key = ioservices/0x7200000000000001:0x20/pvers/
0x7600000000000001:0x6/users/1
value = {"bc": 4096, "object_cnt": 1}
'''
iosservice_items = self.consul.kv.kv_get('ioservices/', recurse=True)
pver_items = {}
if iosservice_items:
for k in iosservice_items:
p_ver = k['Key'].split('/')[3]
if p_ver not in pver_items:
pver_info: PverInfo = motr.get_pver_status(
Fid.parse(p_ver))
pver_items[p_ver] = pver_info
LOG.debug('Received pool version and status: %s', pver_items)
return pver_items
def fn(request):
queue: Queue = Queue(1)
planner.add_command(
status_type(reply_to=queue,
fid=Fid.parse(request.query['pool_fid'])))
return queue.get(timeout=10)
def fn(data: Dict[str, Any]):
fid = Fid.parse(data['fid'])
return a_type(fid)
def fn(request):
queue: Queue = Queue(1)
motr_queue.put(
status_type(reply_to=queue,
fid=Fid.parse(request.query['pool_fid'])))
return queue.get(timeout=10)
def to_profile(k: str, v: Dict[str, Any]) -> Profile:
return Profile(fid=Fid.parse(k),
name=v['name'],
pool_names=v['pools'])
def test_get_nvec_replies_something(mocker, planner, motr, consumer,
consul_util):
def new_kv(key: str, val: str):
return {
'Key': key,
'CreateIndex': 1793,
'ModifyIndex': 1793,
'LockIndex': 0,
'Flags': 0,
'Value': val,
'Session': ''
}
def my_get(key: str, recurse: bool = False, **kwds):
if key == 'm0conf/nodes' and recurse:
return [
new_kv(k, v) for k, v in
[('m0conf/nodes/0x6e00000000000001:0x3/processes'
'/0x7200000000000001:0x15',
json.dumps({
"name": "m0_server",
"state": "offline"
})), ('m0conf/nodes/cmu/processes/21/services/rm', '16'),
('m0conf/nodes/localhost/processes/21/services/rms', '17'),
('m0conf/nodes/0x6e00000000000001:0x3/processes'
'/0x7200000000000001:0x15/services/0x7300000000000001:0x10',
json.dumps({
"name": "rms",
"state": "failed"
})),
('m0conf/nodes/0x6e00000000000001:0x3/processes'
'/0x7200000000000001:0x15/services/0x7300000000000001:0x10'
'/sdevs/0x6400000000000001:0x20',
json.dumps({
"name": "ios",
"state": "failed"
})),
('m0conf/nodes/0x6e00000000000001:0x3/processes'
'/0x7200000000000001:0xa/services/0x7300000000000001:0xc',
json.dumps({
"name": "ios",
"state": "failed"
}))]
]
elif key == 'm0conf/nodes/localhost/processes/7/services/rms':
return new_kv('m0conf/nodes/localhost/processes/7/services/rms',
'17')
elif key == 'm0conf/nodes/0x6e00000000000001:0x3':
return new_kv(
'm0conf/nodes/0x6e00000000000001:0x3',
json.dumps({
"name": "localhost",
"state": "M0_NC_UNKNOWN"
}))
elif key == 'm0conf/sites' and recurse:
return [
new_kv(k, v) for k, v in
[('m0conf/sites/0x5300000000000001:0x1/racks'
'/0x6100000000000001:0x2/encls/0x6500000000000001:0x4'
'/ctrls/0x6300000000000001:0x5',
json.dumps({"state": "M0_NC_UNKNOWN"})),
('m0conf/sites/0x5300000000000001:0x1/racks'
'/0x6100000000000001:0x2/encls/0x6500000000000001:0x4'
'/ctrls/0x6300000000000001:0x6/drives'
'/0x6b00000000000001:0x2d',
json.dumps({"sdev": "0x6400000000000001:0x20"})),
('m0conf/sites/0x5300000000000001:0x1/racks'
'/0x6100000000000001:0x2/encls/0x6500000000000001:0x4'
'/ctrls/0x6300000000000001:0x6',
json.dumps({"state": "M0_NC_UNKNOWN"}))]
]
raise RuntimeError(f'Unexpected call: key={key}, recurse={recurse}')
def my_services(name):
if name == 'confd':
return [{
'Node': 'localhost',
'Service': 'confd',
'ServiceID': '7',
'Address': '192.168.0.28',
'ServiceAddress': '192.168.0.28',
'ServicePort': '12345'
}]
if name == 'hax':
return [{
'Node': 'localhost',
'Service': 'hax',
'ServiceID': '45',
'Address': '192.168.0.28',
'ServiceAddress': '192.168.0.28',
'ServicePort': '667'
}]
raise RuntimeError(f'Unexpected call: name={name}')
patch = mocker.patch.object
patch(consul_util.kv, 'kv_get', side_effect=my_get)
patch(consul_util, 'get_leader_session_no_wait', return_value='localhost')
patch(consul_util, 'get_session_node', return_value='localhost')
patch(consul_util.catalog, 'get_services', side_effect=my_services)
patch(consul_util, 'get_node_health_status', return_value='passing')
patch(consul_util, 'get_service_health', return_value=ServiceHealth.OK)
msg = HaNvecGetEvent(
hax_msg=12,
nvec=[
HaNote(obj_t='SERVICE',
note=HaNoteStruct(
no_id=Fid.parse('0x7300000000000001:0x10').to_c(),
no_state=5)),
HaNote(obj_t='PROCESS',
note=HaNoteStruct(
no_id=Fid.parse('0x7200000000000001:0x15').to_c(),
no_state=5)),
HaNote(obj_t='DRIVE',
note=HaNoteStruct(
no_id=Fid.parse('0x6b00000000000001:0x2d').to_c(),
no_state=5)),
HaNote(obj_t='CONTROLLER',
note=HaNoteStruct(
no_id=Fid.parse('0x6300000000000001:0x5').to_c(),
no_state=5))
])
run_in_consumer(mocker, msg, planner, consumer, motr)
traces = motr._ffi.traces
assert AssertionPlan(
tr_method('ha_nvec_reply')).exists(traces), 'ha_nvec_reply not invoked'