def post_send_ex(agr_obj, send_object, gid_index, port, send_op=None):
qp_type = agr_obj.qp.qp_type
agr_obj.qp.wr_start()
agr_obj.qp.wr_id = 0x123
agr_obj.qp.wr_flags = e.IBV_SEND_SIGNALED
if send_op == e.IBV_QP_EX_WITH_SEND:
agr_obj.qp.wr_send()
elif send_op == e.IBV_QP_EX_WITH_RDMA_WRITE:
agr_obj.qp.wr_rdma_write(agr_obj.rkey, agr_obj.raddr)
elif send_op == e.IBV_QP_EX_WITH_SEND_WITH_IMM:
agr_obj.qp.wr_send_imm(IMM_DATA)
elif send_op == e.IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM:
agr_obj.qp.wr_rdma_write_imm(agr_obj.rkey, agr_obj.raddr, IMM_DATA)
elif send_op == e.IBV_QP_EX_WITH_RDMA_READ:
agr_obj.qp.wr_rdma_read(agr_obj.rkey, agr_obj.raddr)
elif send_op == e.IBV_QP_EX_WITH_ATOMIC_CMP_AND_SWP:
# We're checking the returned value (remote's content), so cmp/swp
# values are of no importance.
agr_obj.qp.wr_atomic_cmp_swp(agr_obj.rkey, agr_obj.raddr, 42, 43)
elif send_op == e.IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD:
agr_obj.qp.wr_atomic_fetch_add(agr_obj.rkey, agr_obj.raddr, 1)
elif send_op == e.IBV_QP_EX_WITH_BIND_MW:
bind_info = MWBindInfo(agr_obj.mr, agr_obj.mr.buf, agr_obj.mr.rkey,
e.IBV_ACCESS_REMOTE_WRITE)
mw = MW(agr_obj.pd, mw_type=e.IBV_MW_TYPE_2)
# A new rkey is needed to be set into bind_info, modify rkey
agr_obj.qp.wr_bind_mw(mw, agr_obj.mr.rkey + 12, bind_info)
agr_obj.qp.wr_send()
if qp_type == e.IBV_QPT_UD:
ah = get_global_ah(agr_obj, gid_index, port)
agr_obj.qp.wr_set_ud_addr(ah, agr_obj.rqpn, agr_obj.UD_QKEY)
if qp_type == e.IBV_QPT_XRC_SEND:
agr_obj.qp.wr_set_xrc_srqn(agr_obj.remote_srqn)
agr_obj.qp.wr_set_sge(send_object)
agr_obj.qp.wr_complete()
def invalidate_mw_type1(self):
"""
Invalidate the MWs by rebind this MW with zero length.
:return: None
"""
for player in [self.server, self.client]:
mw_bind_info = MWBindInfo(player.mr, player.mr.buf, 0, 0)
mw_bind = MWBind(mw_bind_info, e.IBV_SEND_SIGNALED)
player.qp.bind_mw(player.mw, mw_bind)
# Poll the bound MW action request completion.
u.poll_cq(player.cq)
def post_send_ex(agr_obj, send_object, send_op=None, qp_idx=0, ah=None):
qp = agr_obj.qps[qp_idx]
qp_type = qp.qp_type
qp.wr_start()
qp.wr_id = 0x123
qp.wr_flags = e.IBV_SEND_SIGNALED
if send_op == e.IBV_QP_EX_WITH_SEND:
qp.wr_send()
elif send_op == e.IBV_QP_EX_WITH_RDMA_WRITE:
qp.wr_rdma_write(agr_obj.rkey, agr_obj.raddr)
elif send_op == e.IBV_QP_EX_WITH_SEND_WITH_IMM:
qp.wr_send_imm(IMM_DATA)
elif send_op == e.IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM:
qp.wr_rdma_write_imm(agr_obj.rkey, agr_obj.raddr, IMM_DATA)
elif send_op == e.IBV_QP_EX_WITH_RDMA_READ:
qp.wr_rdma_read(agr_obj.rkey, agr_obj.raddr)
elif send_op == e.IBV_QP_EX_WITH_ATOMIC_CMP_AND_SWP:
# We're checking the returned value (remote's content), so cmp/swp
# values are of no importance.
qp.wr_atomic_cmp_swp(agr_obj.rkey, agr_obj.raddr, 42, 43)
elif send_op == e.IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD:
qp.wr_atomic_fetch_add(agr_obj.rkey, agr_obj.raddr, 1)
elif send_op == e.IBV_QP_EX_WITH_BIND_MW:
bind_info = MWBindInfo(agr_obj.mr, agr_obj.mr.buf, agr_obj.mr.rkey,
e.IBV_ACCESS_REMOTE_WRITE)
mw = MW(agr_obj.pd, mw_type=e.IBV_MW_TYPE_2)
# A new rkey is needed to be set into bind_info, modify rkey
qp.wr_bind_mw(mw, agr_obj.mr.rkey + 12, bind_info)
qp.wr_send()
if qp_type == e.IBV_QPT_UD:
qp.wr_set_ud_addr(ah, agr_obj.rqps_num[qp_idx], agr_obj.UD_QKEY)
if isinstance(agr_obj, SRDResources):
qp.wr_set_ud_addr(ah, agr_obj.rqps_num[qp_idx], agr_obj.SRD_QKEY)
if qp_type == e.IBV_QPT_XRC_SEND:
qp.wr_set_xrc_srqn(agr_obj.remote_srqn)
if hasattr(agr_obj, 'remote_dct_num'):
if isinstance(agr_obj, Mlx5DcStreamsRes):
stream_id = agr_obj.generate_stream_id(qp_idx)
agr_obj.check_bad_flow(qp_idx)
qp.wr_set_dc_addr_stream(ah, agr_obj.remote_dct_num, DCT_KEY,
stream_id)
else:
qp.wr_set_dc_addr(ah, agr_obj.remote_dct_num, DCT_KEY)
qp.wr_set_sge(send_object)
qp.wr_complete()
def test_qp_ex_rc_bind_mw(self):
"""
Verify bind memory window operation using the new post_send API.
Instead of checking through regular pingpong style traffic, we'll
do as follows:
- Register an MR with remote write access
- Bind a MW without remote write permission to the MR
- Verify that remote write fails
Since it's a unique flow, it's an integral part of that test rather
than a utility method.
"""
client, server = self.create_players('rc_bind_mw')
client_sge = u.get_send_elements(client, False)[1]
# Create a MW and bind it
server.qp.wr_start()
server.qp.wr_id = 0x123
server.qp.wr_flags = e.IBV_SEND_SIGNALED
bind_info = MWBindInfo(server.mr, server.mr.buf, server.mr.length,
e.IBV_ACCESS_LOCAL_WRITE)
try:
mw = MW(server.pd, mw_type=e.IBV_MW_TYPE_2)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest(
'Memory Window allocation is not supported')
raise ex
new_key = inc_rkey(server.mr.rkey)
server.qp.wr_bind_mw(mw, new_key, bind_info)
server.qp.wr_complete()
u.poll_cq(server.cq)
# Verify that remote write fails
client.qp.wr_start()
client.qp.wr_id = 0x124
client.qp.wr_flags = e.IBV_SEND_SIGNALED
client.qp.wr_rdma_write(new_key, server.mr.buf)
client.qp.wr_set_sge(client_sge)
client.qp.wr_complete()
try:
u.poll_cq(client.cq)
except PyverbsRDMAError as ex:
if ex.error_code != e.IBV_WC_REM_ACCESS_ERR:
raise ex
def __init__(self, dev_name, ib_port, gid_index, mw_type):
"""
Initialize Memory Window resources based on RC resources that include RC
QP.
:param dev_name: Device name to be used
:param ib_port: IB port of the device to use
:param gid_index: Which GID index to use
:param mw_type: The MW type to use
"""
super().__init__(dev_name=dev_name, ib_port=ib_port,
gid_index=gid_index)
self.mw_type = mw_type
access = e.IBV_ACCESS_REMOTE_WRITE | e.IBV_ACCESS_LOCAL_WRITE
self.mw_bind_info = MWBindInfo(self.mr, self.mr.buf, self.msg_size,
access)
self.mw_bind = MWBind(self.mw_bind_info, e.IBV_SEND_SIGNALED)
try:
self.mw = MW(self.pd, self.mw_type)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create MW is not supported')
raise ex