def test_nvmeerror_inject(self):
self.assertIsNotNone(ssh, "Can't connect node by ssh")
# "dev-nvme-0" is the first NVME dev.
dev_sn = conf["compute"]["storage_backend"][1]["serial"]
dev = self.get_nvme_dev(dev_sn)
self.assertIsNotNone(dev, "Can't found nvme device for sn {}".format(dev_sn))
s = UnixSocket(path)
s.connect()
s.recv()
payload_enable_qmp = {
"execute": "qmp_capabilities"
}
s.send(json.dumps(payload_enable_qmp))
s.recv()
payload_error_inject = {
"execute": "nvme-status-code-error-inject",
"arguments": {
"count": 65536,
"opcode": "rw",
"id": "dev-nvme-0",
"nsid": 1,
"status_field": {
"dnr": True,
"more": True
}
}
}
payload_nvmeclear = {
'execute': 'nvme-status-code-error-inject',
'arguments': {
'count': 0,
'opcode': 'rw',
'id': 'dev-nvme-0',
'nsid': 0,
'status_field': {
'sc': 0,
'sct': 0,
'dnr': True,
'more': True
}
}
}
# Redirect stderr to stdout since 'Success' is printed to stderr sometimes"
cmd = "nvme read {} -z 3008 -a 128 2>&1".format(dev)
for cmd_error_inject in error_inject_list:
payload_error_inject['arguments']['status_field']['sc'] = cmd_error_inject[0]
payload_error_inject['arguments']['status_field']['sct'] = cmd_error_inject[1]
s.send(json.dumps(payload_error_inject))
s.recv()
output = helper.ssh_exec(ssh, cmd)
self.assertNotIn("Success", output, "error of %s inject failed" % cmd_error_inject[2])
s.send(json.dumps(payload_nvmeclear))
s.recv()
output = helper.ssh_exec(ssh, cmd)
self.assertIn("Success", output, "clear error failed")
s.close()
def test_scsi_write_to_disk(self):
global ssh1
global ssh2
drive_a = [
x.split(" ")[-1]
for x in helper.ssh_exec(ssh1, "lsscsi -g").strip().split("\n")
if "B29C" in x
]
# write special pattern to drive
data = "\\xff\\x01\\x02\\x03\\x04\\x05\\x06\\x07"
for _ in range(0, 504 / 8):
data = data + "\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00"
data = data + "\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\xff"
# write 520 bytes to 2nd sector.
cmd = "echo -ne '{0}' | sg_dd of={1} bs=520 seek=1".format(
data, drive_a[0])
helper.ssh_exec(ssh1, cmd)
drive_b = [
x.split(" ")[-1]
for x in helper.ssh_exec(ssh2, "lsscsi -g").strip().split("\n")
if "B29C" in x
]
cmd = "sg_dd if={} bs=520 count=3 | hd".format(drive_b[0])
lines = run_cmd(cmd, nodes_ip[1])
self.assertIn(
"00000200 00 00 00 00 00 00 00 00 ff 01 02 03 04 05 06 07",
lines, "sector start error")
self.assertIn(
"00000400 00 00 00 00 00 00 00 00 01 02 03 04 05 06 07 ff",
lines, "sector end error")
def test_status_code_error_inject(self):
cmd_nic_down = {
"execute": "set_link",
"arguments": {
"name": "e1000.1",
"up": False
}
}
cmd_nic_up = {
"execute": "set_link",
"arguments": {
"name": "e1000.1",
"up": True
}
}
# defore link down interface ,use ssh do some prepare works
nic_name = get_interface_name()
print("Found target nic: {}".format(nic_name))
output = ssh_exec(ssh, "dhclient {}".format(nic_name))
output = ssh_exec(ssh, "ifconfig {}".format(nic_name))
print(output)
self.assertIn(",RUNNING,", output,
"NIC {} is not up yet".format(nic_name))
# down interface and get status
s.send(json.dumps(cmd_nic_down))
s.recv()
# try 10 times,
for _ in range(0, 10):
output = ssh_exec(ssh, "ifconfig {}".format(nic_name))
if ",RUNNING," not in output:
break
sleep(1)
else:
print(output)
self.fail("NIC {} is not down".format(nic_name))
# set interface up and get status
s.send(json.dumps(cmd_nic_up))
s.recv()
for _ in range(0, 10):
output = ssh_exec(ssh, "ifconfig {}".format(nic_name))
if ",RUNNING," in output:
break
sleep(1)
else:
print(output)
self.fail("NIC {} is not up again".format(nic_name))
def get_nvme_dev(self, dev_sn):
# find correct dev id by sn
nvme_list = helper.ssh_exec(ssh, "nvme list -o json")
nvme_object = json.loads(nvme_list)
for nvme_dev in nvme_object["Devices"]:
if nvme_dev["SerialNumber"] == dev_sn:
return nvme_dev["DevicePath"]
return None
def test_scsi_write_to_disk(self):
global ssh1
global ssh2
drive_a = [x.split(" ")[-1] for x in helper.ssh_exec(ssh1, "lsscsi -g").strip().split("\n") if "B29C" in x]
# write special pattern to drive
data = "\\xff\\x01\\x02\\x03\\x04\\x05\\x06\\x07"
for _ in range(0, 504 / 8):
data = data + "\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00"
data = data + "\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\xff"
# write 520 bytes to 2nd sector.
cmd = "echo -ne '{0}' | sg_dd of={1} bs=520 seek=1".format(data, drive_a[0])
helper.ssh_exec(ssh1, cmd)
drive_b = [x.split(" ")[-1] for x in helper.ssh_exec(ssh2, "lsscsi -g").strip().split("\n") if "B29C" in x]
cmd = "sg_dd if={} bs=520 count=3 | hd".format(drive_b[0])
lines = run_cmd(cmd, nodes_ip[1])
self.assertIn("00000200 00 00 00 00 00 00 00 00 ff 01 02 03 04 05 06 07", lines, "sector start error")
self.assertIn("00000400 00 00 00 00 00 00 00 00 01 02 03 04 05 06 07 ff", lines, "sector end error")
def get_interface_name():
inface = "lo"
ret = ssh_exec(
ssh,
"for f in /sys/class/net/*/address; do echo -n $f' '; cat $f; done")
for line in ret.split('\n'):
if target_mac in line:
inface = line.split('/')[4]
break
return inface
def run_cmd(cmd):
return helper.ssh_exec(ssh, cmd)
def run_cmd(cmd):
return helper.ssh_exec(ssh, cmd)
def run_cmd(cmd, ip):
if ip == nodes_ip[0]:
ssh = ssh1
else:
ssh = ssh2
return helper.ssh_exec(ssh, cmd)
def run_cmd(cmd, ip):
if ip == nodes_ip[0]:
ssh = ssh1
else:
ssh = ssh2
return helper.ssh_exec(ssh, cmd)
def run_cmd(cmd):
global ssh
return helper.ssh_exec(ssh, cmd)