def _ext_qp_client_traffic(self):
self.cm_res.remote_dct_num = self.cm_res.remote_qpn
_, send_wr = u.get_send_elements(self.cm_res, self.cm_res.passive)
ah = AH(self.cm_res.cmid.pd, attr=self.cm_res.remote_ah)
self.syncer.wait()
for send_idx in range(self.cm_res.num_msgs):
dci_idx = send_idx % len(self.cm_res.qps)
u.post_send_ex(self.cm_res,
send_wr,
e.IBV_QP_EX_WITH_SEND,
ah=ah,
qp_idx=dci_idx)
u.poll_cq(self.cm_res.cq)
def ts_traffic(self):
"""
Run RDMA traffic and read the completions timestamps.
"""
s_recv_wr = u.get_recv_wr(self.server)
u.post_recv(self.server, s_recv_wr)
if self.qp_type == e.IBV_QPT_RAW_PACKET:
c_send_wr, _, _ = u.get_send_elements_raw_qp(self.client)
else:
c_send_wr, _ = u.get_send_elements(self.client, False)
u.send(self.client, c_send_wr, e.IBV_WR_SEND, False, 0)
self.client.timestamp = self.poll_cq_ex_ts(self.client.scq, ts_type=self.send_ts)
self.server.timestamp = self.poll_cq_ex_ts(self.server.rcq, ts_type=self.recv_ts)
def _ext_qp_client_traffic(self):
"""
RDMACM client side traffic function which sends and receives a message,
and then validates the received message. This traffic method uses the CM
external QP and CQ for send, recv and get_completion.
:return: None
"""
recv_wr = get_recv_wr(self.cm_res)
self.syncer.wait()
for _ in range(self.cm_res.num_msgs):
send_wr = get_send_elements(self.cm_res, self.cm_res.passive)[0]
self.cm_res.qp.post_send(send_wr)
poll_cq(self.cm_res.cq)
self.cm_res.qp.post_recv(recv_wr)
poll_cq(self.cm_res.cq)
msg_received = self.cm_res.mr.read(self.cm_res.msg_size, 0)
validate(msg_received, self.cm_res.passive, self.cm_res.msg_size)
def test_qp_ex_rc_bind_mw(self):
"""
Verify bind memory window operation using the new post_send API.
Instead of checking through regular pingpong style traffic, we'll
do as follows:
- Register an MR with remote write access
- Bind a MW without remote write permission to the MR
- Verify that remote write fails
Since it's a unique flow, it's an integral part of that test rather
than a utility method.
"""
client, server = self.create_players('rc_bind_mw')
client_sge = u.get_send_elements(client, False)[1]
# Create a MW and bind it
server.qp.wr_start()
server.qp.wr_id = 0x123
server.qp.wr_flags = e.IBV_SEND_SIGNALED
bind_info = MWBindInfo(server.mr, server.mr.buf, server.mr.length,
e.IBV_ACCESS_LOCAL_WRITE)
try:
mw = MW(server.pd, mw_type=e.IBV_MW_TYPE_2)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest(
'Memory Window allocation is not supported')
raise ex
new_key = inc_rkey(server.mr.rkey)
server.qp.wr_bind_mw(mw, new_key, bind_info)
server.qp.wr_complete()
u.poll_cq(server.cq)
# Verify that remote write fails
client.qp.wr_start()
client.qp.wr_id = 0x124
client.qp.wr_flags = e.IBV_SEND_SIGNALED
client.qp.wr_rdma_write(new_key, server.mr.buf)
client.qp.wr_set_sge(client_sge)
client.qp.wr_complete()
try:
u.poll_cq(client.cq)
except PyverbsRDMAError as ex:
if ex.error_code != e.IBV_WC_REM_ACCESS_ERR:
raise ex
def full_sq_bad_flow(self):
"""
Check post_send while qp's sq is full.
- Find qp's sq length
- Fill the qp with work requests until overflow
"""
qp_idx = 0
send_op = e.IBV_QP_EX_WITH_SEND
ah = u.get_global_ah(self.client, self.gid_index, self.ib_port)
qp_attr, _ = self.client.qps[qp_idx].query(e.IBV_QP_CAP)
max_send_wr = qp_attr.cap.max_send_wr
with self.assertRaises(PyverbsRDMAError) as ex:
for _ in range(max_send_wr + 1):
_, c_sg = u.get_send_elements(self.client, False)
u.send(self.client,
c_sg,
send_op,
new_send=True,
qp_idx=qp_idx,
ah=ah)
self.assertEqual(ex.exception.error_code, errno.ENOMEM)
def test_resize_cq(self):
"""
Test resize CQ, start with specific value and then increase and decrease
the CQ size. The test also check bad flow of decrease the CQ size when
there are more completions on it than the new value.
"""
self.create_players(CQUDResources, cq_depth=3)
# Decrease the CQ size.
new_cq_size = 1
try:
self.client.cq.resize(new_cq_size)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Resize CQ is not supported')
raise ex
self.assertTrue(
self.client.cq.cqe >= new_cq_size,
f'The actual CQ size ({self.client.cq.cqe}) is less '
'than guaranteed ({new_cq_size})')
# Increase the CQ size.
new_cq_size = 7
self.client.cq.resize(new_cq_size)
self.assertTrue(
self.client.cq.cqe >= new_cq_size,
f'The actual CQ size ({self.client.cq.cqe}) is less '
'than guaranteed ({new_cq_size})')
# Fill the CQ entries except one for avoid cq_overrun warnings.
send_wr, _ = u.get_send_elements(self.client, False)
ah_client = u.get_global_ah(self.client, self.gid_index, self.ib_port)
for i in range(self.client.cq.cqe - 1):
u.send(self.client, send_wr, ah=ah_client)
# Decrease the CQ size to less than the CQ unpolled entries.
new_cq_size = 1
with self.assertRaises(PyverbsRDMAError) as ex:
self.client.cq.resize(new_cq_size)
self.assertEqual(ex.exception.error_code, errno.EINVAL)
def traffic_with_bad_flow(client, server, iters, gid_idx, port):
"""
Runs basic traffic with bad flow between two sides
:param client: client side, clients base class is BaseTraffic
:param server: server side, servers base class is BaseTraffic
:param iters: number of traffic iterations
:param gid_idx: local gid index
:param port: IB port
:return: None
"""
import tests.utils as u
send_op = e.IBV_QP_EX_WITH_SEND
ah_client = u.get_global_ah(client, gid_idx, port)
s_recv_wr = u.get_recv_wr(server)
c_recv_wr = u.get_recv_wr(client)
for qp_idx in range(server.qp_count):
# Prepare the receive queue with RecvWR
u.post_recv(client, c_recv_wr, qp_idx=qp_idx)
u.post_recv(server, s_recv_wr, qp_idx=qp_idx)
read_offset = 0
for _ in range(iters):
for qp_idx in range(server.qp_count):
_, c_send_object = u.get_send_elements(client, False)
u.send(client, c_send_object, send_op, True, qp_idx, ah_client,
False)
try:
u.poll_cq(client.cq)
except PyverbsError as ex:
if client.bad_flow_handling(qp_idx, ex, True):
continue
raise ex
u.poll_cq(server.cq)
u.post_recv(server, s_recv_wr, qp_idx=qp_idx)
msg_received = server.mr.read(server.msg_size, read_offset)
u.validate(msg_received, True, server.msg_size)
client.check_after_traffic()