def _ext_qp_server_traffic(self):
recv_wr = u.get_recv_wr(self.cm_res)
for _ in range(self.cm_res.num_msgs):
u.post_recv(self.cm_res, recv_wr)
self.syncer.wait()
for _ in range(self.cm_res.num_msgs):
u.poll_cq(self.cm_res.cq)
def reg_mr_interleaved(self, configure_mkey=False):
"""
Register an interleaved memory layout using the player's mkeys.
:param configure_mkey: Use the mkey configuration API.
"""
for player in [self.server, self.client]:
player.qp.wr_start()
player.qp.wr_flags = e.IBV_SEND_SIGNALED | e.IBV_SEND_INLINE
mr_interleaved_1 = Mlx5MrInterleaved(addr=player.mr.buf,
bytes_count=8,
bytes_skip=2,
lkey=player.mr.lkey)
mr_interleaved_2 = Mlx5MrInterleaved(addr=player.mr.buf + 64,
bytes_count=8,
bytes_skip=2,
lkey=player.mr.lkey)
mr_interleaved_lst = [mr_interleaved_1, mr_interleaved_2]
mkey_access = e.IBV_ACCESS_LOCAL_WRITE
if configure_mkey:
player.qp.wr_mkey_configure(player.mkey, 2, Mlx5MkeyConfAttr())
player.qp.wr_set_mkey_access_flags(mkey_access)
player.qp.wr_set_mkey_layout_interleaved(3, mr_interleaved_lst)
else:
player.qp.wr_mr_interleaved(
player.mkey,
e.IBV_ACCESS_LOCAL_WRITE,
repeat_count=3,
mr_interleaved_lst=mr_interleaved_lst)
player.qp.wr_complete()
u.poll_cq(player.cq)
def reg_client_mkey(self, signature=False):
"""
Configure an mkey with crypto attributes.
:param signature: True if signature configuration requested.
"""
num_of_configuration = 4 if signature else 3
for mkey in [self.client.wire_enc_mkey, self.client.mem_enc_mkey]:
self.client.qp.wr_start()
self.client.qp.wr_flags = e.IBV_SEND_SIGNALED | e.IBV_SEND_INLINE
offset = 0 if mkey == self.client.wire_enc_mkey else self.client.msg_size/2
sge = SGE(self.client.mr.buf + offset, self.client.msg_size/2, self.client.mr.lkey)
self.client.qp.wr_mkey_configure(mkey, num_of_configuration, Mlx5MkeyConfAttr())
self.client.qp.wr_set_mkey_access_flags(e.IBV_ACCESS_LOCAL_WRITE)
self.client.qp.wr_set_mkey_layout_list([sge])
if signature:
self.configure_mkey_signature()
initial_tweak = struct.pack('!2Q', int(0), int(0))
encrypt_on_tx = mkey == self.client.wire_enc_mkey
sign_crypto_order = dve.MLX5DV_SIGNATURE_CRYPTO_ORDER_SIGNATURE_BEFORE_CRYPTO_ON_TX
crypto_attr = Mlx5CryptoAttr(crypto_standard=dve.MLX5DV_CRYPTO_STANDARD_AES_XTS,
encrypt_on_tx=encrypt_on_tx,
signature_crypto_order=sign_crypto_order,
data_unit_size=dve.MLX5DV_BLOCK_SIZE_512,
dek=self.dek, initial_tweak=initial_tweak)
self.client.qp.wr_set_mkey_crypto(crypto_attr)
self.client.qp.wr_complete()
u.poll_cq(self.client.cq)
def reg_mr_sig_t10dif(self):
"""
Register the player's mkeys with T10DIF signature on the wire domain.
"""
for player in [self.server, self.client]:
player.qp.wr_start()
player.qp.wr_flags = e.IBV_SEND_SIGNALED | e.IBV_SEND_INLINE
sge = SGE(player.mr.buf, 512, player.mr.lkey)
player.qp.wr_mkey_configure(player.mkey, 3, Mlx5MkeyConfAttr())
player.qp.wr_set_mkey_access_flags(e.IBV_ACCESS_LOCAL_WRITE)
player.qp.wr_set_mkey_layout_list([sge])
t10dif_flags = (dve.MLX5DV_SIG_T10DIF_FLAG_REF_REMAP
| dve.MLX5DV_SIG_T10DIF_FLAG_APP_ESCAPE
| dve.MLX5DV_SIG_T10DIF_FLAG_APP_REF_ESCAPE)
sig_t10dif = Mlx5SigT10Dif(bg_type=dve.MLX5DV_SIG_T10DIF_CRC,
bg=0xFFFF,
app_tag=0xABCD,
ref_tag=0x01234567,
flags=t10dif_flags)
sig_type = dve.MLX5DV_SIG_TYPE_T10DIF
block_size = dve.MLX5DV_BLOCK_SIZE_512
sig_block_domain = Mlx5SigBlockDomain(sig_type=sig_type,
dif=sig_t10dif,
block_size=block_size)
check_mask = (dve.MLX5DV_SIG_MASK_T10DIF_GUARD
| dve.MLX5DV_SIG_MASK_T10DIF_APPTAG
| dve.MLX5DV_SIG_MASK_T10DIF_REFTAG)
sig_attr = Mlx5SigBlockAttr(wire=sig_block_domain,
check_mask=check_mask)
player.qp.wr_set_mkey_sig_block(sig_attr)
player.qp.wr_complete()
u.poll_cq(player.cq)
def invalidate_mkeys(self):
"""
Invalidate the players mkey.
"""
for player in [self.server, self.client]:
inv_send_wr = SendWR(opcode=e.IBV_WR_LOCAL_INV)
inv_send_wr.imm_data = player.mkey.lkey
player.qp.post_send(inv_send_wr)
u.poll_cq(player.cq)
def bind_mw_type_1(self):
self.server.qp.bind_mw(self.server.mw, self.server.mw_bind)
self.client.qp.bind_mw(self.client.mw, self.client.mw_bind)
# Poll the bind MW action completion.
u.poll_cq(self.server.cq)
u.poll_cq(self.client.cq)
self.server.rkey = self.client.mw.rkey
self.server.remote_addr = self.client.mr.buf
self.client.rkey = self.server.mw.rkey
self.client.remote_addr = self.server.mr.buf
def invalidate_mw_type1(self):
"""
Invalidate the MWs by rebind this MW with zero length.
:return: None
"""
for player in [self.server, self.client]:
mw_bind_info = MWBindInfo(player.mr, player.mr.buf, 0, 0)
mw_bind = MWBind(mw_bind_info, e.IBV_SEND_SIGNALED)
player.qp.bind_mw(player.mw, mw_bind)
# Poll the bound MW action request completion.
u.poll_cq(player.cq)
def _client_traffic_with_ext_qp(agr_obj, syncer):
recv_wr = get_recv_wr(agr_obj)
syncer.wait()
for _ in range(agr_obj.num_msgs):
send_wr = get_send_element(agr_obj, agr_obj.is_server)[0]
agr_obj.qp.post_send(send_wr)
poll_cq(agr_obj.cq)
agr_obj.qp.post_recv(recv_wr)
poll_cq(agr_obj.cq)
msg_received = agr_obj.mr.read(agr_obj.msg_size, 0)
validate(msg_received, agr_obj.is_server, agr_obj.msg_size)
def _ext_qp_client_traffic(self):
self.cm_res.remote_dct_num = self.cm_res.remote_qpn
_, send_wr = u.get_send_elements(self.cm_res, self.cm_res.passive)
ah = AH(self.cm_res.cmid.pd, attr=self.cm_res.remote_ah)
self.syncer.wait()
for send_idx in range(self.cm_res.num_msgs):
dci_idx = send_idx % len(self.cm_res.qps)
u.post_send_ex(self.cm_res,
send_wr,
e.IBV_QP_EX_WITH_SEND,
ah=ah,
qp_idx=dci_idx)
u.poll_cq(self.cm_res.cq)
def reg_mr_sig_pipelining_server(self):
"""
Register mkey without signature.
"""
self.server.qp.wr_start()
self.server.qp.wr_flags = e.IBV_SEND_SIGNALED | e.IBV_SEND_INLINE
sge = SGE(self.server.mr.buf, 512, self.server.mr.lkey)
self.server.qp.wr_mkey_configure(self.server.mkey, 2,
Mlx5MkeyConfAttr())
self.server.qp.wr_set_mkey_access_flags(e.IBV_ACCESS_LOCAL_WRITE
| e.IBV_ACCESS_REMOTE_WRITE)
self.server.qp.wr_set_mkey_layout_list([sge])
self.server.qp.wr_complete()
u.poll_cq(self.server.cq)
def bind_mw_type_2(self):
client_send_wr = SendWR(opcode=e.IBV_WR_BIND_MW)
client_send_wr.set_bind_wr(self.client.mw, self.client.mw_bind_info)
server_send_wr = SendWR(opcode=e.IBV_WR_BIND_MW)
server_send_wr.set_bind_wr(self.server.mw, self.server.mw_bind_info)
self.server.qp.post_send(server_send_wr)
self.client.qp.post_send(client_send_wr)
# Poll the bind MW WR.
u.poll_cq(self.server.cq)
u.poll_cq(self.client.cq)
self.server.rkey = client_send_wr.rkey
self.server.remote_addr = self.client.mr.buf
self.client.rkey = server_send_wr.rkey
self.client.remote_addr = self.server.mr.buf
def invalidate_mw_type2_local(self):
"""
Invalidate the MWs by post invalidation send WR from the local QP.
:return: None
"""
inv_send_wr = SendWR(opcode=e.IBV_WR_LOCAL_INV)
inv_send_wr.imm_data = self.server.rkey
self.client.qp.post_send(inv_send_wr)
inv_send_wr = SendWR(opcode=e.IBV_WR_LOCAL_INV)
inv_send_wr.imm_data = self.client.rkey
self.server.qp.post_send(inv_send_wr)
# Poll the invalidate MW WR.
u.poll_cq(self.server.cq)
u.poll_cq(self.client.cq)
def traffic(self):
"""
Perform RC traffic using the configured mkeys.
"""
if self.validate_data:
self.prepare_validate_data()
for _ in range(self.iters):
self.server.qp.post_recv(self.get_recv_wr(self.server, wire_encryption=True))
self.server.qp.post_recv(self.get_recv_wr(self.server, wire_encryption=False))
self.client.qp.post_send(self.get_send_wr(self.client, wire_encryption=True))
self.client.qp.post_send(self.get_send_wr(self.client, wire_encryption=False))
u.poll_cq(self.client.cq, count=2)
u.poll_cq(self.server.cq, count=2)
if self.validate_data:
self.validate_crypto_data()
def test_odp_ud_traffic(self):
client, server = self.create_players(OdpUD)
# Implement the traffic here because OdpUD uses two different MRs for
# send and recv.
recv_sge = SGE(server.recv_mr.buf, server.msg_size + GRH_SIZE,
server.recv_mr.lkey)
server_recv_wr = RecvWR(sg=[recv_sge], num_sge=1)
send_sge = SGE(client.send_mr.buf + GRH_SIZE, client.msg_size,
client.send_mr.lkey)
client_send_wr = SendWR(num_sge=1, sg=[send_sge])
for i in range(self.iters):
server.qp.post_recv(server_recv_wr)
post_send(client, client_send_wr, self.gid_index, self.ib_port)
poll_cq(client.cq)
poll_cq(server.cq)
def _ext_qp_client_traffic(self):
"""
RDMACM client side traffic function which sends and receives a message,
and then validates the received message. This traffic method uses the CM
external QP and CQ for send, recv and get_completion.
:return: None
"""
recv_wr = get_recv_wr(self.cm_res)
self.syncer.wait()
for _ in range(self.cm_res.num_msgs):
send_wr = get_send_elements(self.cm_res, self.cm_res.passive)[0]
self.cm_res.qp.post_send(send_wr)
poll_cq(self.cm_res.cq)
self.cm_res.qp.post_recv(recv_wr)
poll_cq(self.cm_res.cq)
msg_received = self.cm_res.mr.read(self.cm_res.msg_size, 0)
validate(msg_received, self.cm_res.passive, self.cm_res.msg_size)
def test_mkey_sig_pipelining_bad(self):
"""
Test the bad signature pipelining scenario.
"""
self.create_players(
Mlx5MkeyResources,
dv_send_ops_flags=dve.MLX5DV_QP_EX_WITH_MKEY_CONFIGURE,
mkey_create_flags=dve.MLX5DV_MKEY_INIT_ATTR_FLAGS_INDIRECT
| dve.MLX5DV_MKEY_INIT_ATTR_FLAGS_BLOCK_SIGNATURE,
dv_qp_create_flags=dve.MLX5DV_QP_CREATE_DISABLE_SCATTER_TO_CQE
| dve.MLX5DV_QP_CREATE_SIG_PIPELINING)
self.reg_mr_sig_pipelining_client(check_mask=dve.MLX5DV_SIG_MASK_CRC32)
self.reg_mr_sig_pipelining_server()
self.build_traffic_elements_sig_pipelining()
self.server.qp.post_recv(self.server_resp_wr)
self.client.qp.post_send(self.client_data_wr)
self.client.qp.post_send(self.client_resp_wr)
# Expect SQ_DRAINED event
event = self.client.ctx.get_async_event()
event.ack()
self.assertEqual(event.event_type, e.IBV_EVENT_SQ_DRAINED)
# No completion is expected on the client side
nc, _ = self.client.cq.poll(1)
self.assertEqual(nc, 0)
# No completion is expected on the server side
nc, _ = self.server.cq.poll(1)
self.assertEqual(nc, 0)
self.check_mkey(self.client, dve.MLX5DV_MKEY_SIG_BLOCK_BAD_GUARD)
self.check_mkey(self.server)
# Cancel and repost response WR
canceled_count = self.client.qp.cancel_posted_send_wrs(1)
self.assertEqual(canceled_count, 1)
self.client.qp.post_send(self.client_resp_wr)
# Move QP back to RTS and receive completions
self.client.qp.modify(
QPAttr(qp_state=e.IBV_QPS_RTS, cur_qp_state=e.IBV_QPS_SQD),
e.IBV_QP_STATE | e.IBV_QP_CUR_STATE)
u.poll_cq(self.client.cq)
u.poll_cq(self.server.cq)
def test_qp_ex_rc_bind_mw(self):
"""
Verify bind memory window operation using the new post_send API.
Instead of checking through regular pingpong style traffic, we'll
do as follows:
- Register an MR with remote write access
- Bind a MW without remote write permission to the MR
- Verify that remote write fails
Since it's a unique flow, it's an integral part of that test rather
than a utility method.
"""
client, server = self.create_players('rc_bind_mw')
client_sge = u.get_send_elements(client, False)[1]
# Create a MW and bind it
server.qp.wr_start()
server.qp.wr_id = 0x123
server.qp.wr_flags = e.IBV_SEND_SIGNALED
bind_info = MWBindInfo(server.mr, server.mr.buf, server.mr.length,
e.IBV_ACCESS_LOCAL_WRITE)
try:
mw = MW(server.pd, mw_type=e.IBV_MW_TYPE_2)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest(
'Memory Window allocation is not supported')
raise ex
new_key = inc_rkey(server.mr.rkey)
server.qp.wr_bind_mw(mw, new_key, bind_info)
server.qp.wr_complete()
u.poll_cq(server.cq)
# Verify that remote write fails
client.qp.wr_start()
client.qp.wr_id = 0x124
client.qp.wr_flags = e.IBV_SEND_SIGNALED
client.qp.wr_rdma_write(new_key, server.mr.buf)
client.qp.wr_set_sge(client_sge)
client.qp.wr_complete()
try:
u.poll_cq(client.cq)
except PyverbsRDMAError as ex:
if ex.error_code != e.IBV_WC_REM_ACCESS_ERR:
raise ex
def reg_mr_list(self, configure_mkey=False):
"""
Register a list of SGEs using the player's mkeys.
:param configure_mkey: If True, use the mkey configuration API.
"""
for player in [self.server, self.client]:
player.qp.wr_start()
player.qp.wr_flags = e.IBV_SEND_SIGNALED | e.IBV_SEND_INLINE
sge_1 = SGE(player.mr.buf, 8, player.mr.lkey)
sge_2 = SGE(player.mr.buf + 64, 8, player.mr.lkey)
if configure_mkey:
player.qp.wr_mkey_configure(player.mkey, 2, Mlx5MkeyConfAttr())
player.qp.wr_set_mkey_access_flags(e.IBV_ACCESS_LOCAL_WRITE)
player.qp.wr_set_mkey_layout_list([sge_1, sge_2])
else:
player.qp.wr_mr_list(player.mkey,
e.IBV_ACCESS_LOCAL_WRITE,
sge_list=[sge_1, sge_2])
player.qp.wr_complete()
u.poll_cq(player.cq)
def test_mkey_sig_pipelining_good(self):
"""
Test the good signature pipelining scenario.
"""
self.create_players(
Mlx5MkeyResources,
dv_send_ops_flags=dve.MLX5DV_QP_EX_WITH_MKEY_CONFIGURE,
mkey_create_flags=dve.MLX5DV_MKEY_INIT_ATTR_FLAGS_INDIRECT
| dve.MLX5DV_MKEY_INIT_ATTR_FLAGS_BLOCK_SIGNATURE,
dv_qp_create_flags=dve.MLX5DV_QP_CREATE_DISABLE_SCATTER_TO_CQE
| dve.MLX5DV_QP_CREATE_SIG_PIPELINING)
self.reg_mr_sig_pipelining_client()
self.reg_mr_sig_pipelining_server()
self.build_traffic_elements_sig_pipelining()
self.server.qp.post_recv(self.server_resp_wr)
self.client.qp.post_send(self.client_data_wr)
self.client.qp.post_send(self.client_resp_wr)
u.poll_cq(self.client.cq)
u.poll_cq(self.server.cq)
def reg_mr_sig_err(self):
"""
Register the player's mkeys with an SGE and CRC32 signature on
the memory domain. Data transport operation with these MKEYs will cause
a signature error because the test does not fill out the signature in
the memory buffer.
"""
sig_crc = Mlx5SigCrc(crc_type=dve.MLX5DV_SIG_CRC_TYPE_CRC32,
seed=0xFFFFFFFF)
block_size = dve.MLX5DV_BLOCK_SIZE_512
sig_block_domain = Mlx5SigBlockDomain(sig_type=dve.MLX5DV_SIG_TYPE_CRC,
crc=sig_crc,
block_size=block_size)
sig_attr = Mlx5SigBlockAttr(mem=sig_block_domain,
check_mask=dve.MLX5DV_SIG_MASK_CRC32)
# Configure the mkey on the server side
self.server.qp.wr_start()
self.server.qp.wr_flags = e.IBV_SEND_SIGNALED | e.IBV_SEND_INLINE
sge = SGE(self.server.mr.buf, 512, self.server.mr.lkey)
self.server.qp.wr_mkey_configure(self.server.mkey, 2,
Mlx5MkeyConfAttr())
self.server.qp.wr_set_mkey_access_flags(e.IBV_ACCESS_LOCAL_WRITE)
self.server.qp.wr_set_mkey_layout_list([sge])
self.server.qp.wr_complete()
u.poll_cq(self.server.cq)
# Configure the mkey on the client side
self.client.qp.wr_start()
self.client.qp.wr_flags = e.IBV_SEND_SIGNALED | e.IBV_SEND_INLINE
sge = SGE(self.client.mr.buf, 512 + 4, self.client.mr.lkey)
self.client.qp.wr_mkey_configure(self.client.mkey, 3,
Mlx5MkeyConfAttr())
self.client.qp.wr_set_mkey_access_flags(e.IBV_ACCESS_LOCAL_WRITE)
self.client.qp.wr_set_mkey_layout_list([sge])
self.client.qp.wr_set_mkey_sig_block(sig_attr)
self.client.qp.wr_complete()
u.poll_cq(self.client.cq)
def traffic(self, sge_size, exp_buffer):
"""
Perform RC traffic using the mkey.
:param sge_size: The sge size using the mkey.
:param exp_buffer: The expected result of the receive buffer after
the traffic operation.
"""
self.build_traffic_elements(sge_size)
self.server.qp.post_recv(self.server_recv_wr)
for _ in range(self.iters):
self.server.mr.write('s' * self.server.msg_size,
self.server.msg_size)
self.client.mr.write('c' * self.client.msg_size,
self.client.msg_size)
self.client.qp.post_send(self.client_send_wr)
u.poll_cq(self.client.cq)
u.poll_cq(self.server.cq)
self.server.qp.post_recv(self.server_recv_wr)
act_buffer = self.server.mr.read(len(exp_buffer), 0).decode()
if act_buffer != exp_buffer:
raise PyverbsError('Data validation failed: expected '
f'{exp_buffer}, received {act_buffer}')
def reg_mr_sig_pipelining_client(self, check_mask=0):
"""
Register mkey with CRC32 signature in memory domain and no
signature in wire domain.
:param check_mask: The mask for the signature checking.
"""
self.client.qp.wr_start()
self.client.qp.wr_flags = e.IBV_SEND_SIGNALED | e.IBV_SEND_INLINE
# Add 4 bytes for CRC32 signature
sge = SGE(self.client.mr.buf, 512 + 4, self.client.mr.lkey)
self.client.qp.wr_mkey_configure(self.client.mkey, 3,
Mlx5MkeyConfAttr())
self.client.qp.wr_set_mkey_access_flags(e.IBV_ACCESS_LOCAL_WRITE)
self.client.qp.wr_set_mkey_layout_list([sge])
sig = Mlx5SigCrc(crc_type=dve.MLX5DV_SIG_CRC_TYPE_CRC32)
sig_domain = Mlx5SigBlockDomain(sig_type=dve.MLX5DV_SIG_TYPE_CRC,
crc=sig,
block_size=dve.MLX5DV_BLOCK_SIZE_512)
sig_attr = Mlx5SigBlockAttr(mem=sig_domain, check_mask=check_mask)
self.client.qp.wr_set_mkey_sig_block(sig_attr)
self.client.qp.wr_complete()
u.poll_cq(self.client.cq)
def reg_mr_sig_crc(self):
"""
Register the player's mkeys with CRC32 signature on the wire domain.
"""
for player in [self.server, self.client]:
player.qp.wr_start()
player.qp.wr_flags = e.IBV_SEND_SIGNALED | e.IBV_SEND_INLINE
sge = SGE(player.mr.buf, 512, player.mr.lkey)
player.qp.wr_mkey_configure(player.mkey, 3, Mlx5MkeyConfAttr())
player.qp.wr_set_mkey_access_flags(e.IBV_ACCESS_LOCAL_WRITE)
player.qp.wr_set_mkey_layout_list([sge])
sig_crc = Mlx5SigCrc(crc_type=dve.MLX5DV_SIG_CRC_TYPE_CRC32,
seed=0xFFFFFFFF)
sig_block_domain = Mlx5SigBlockDomain(
sig_type=dve.MLX5DV_SIG_TYPE_CRC,
crc=sig_crc,
block_size=dve.MLX5DV_BLOCK_SIZE_512)
sig_attr = Mlx5SigBlockAttr(wire=sig_block_domain,
check_mask=dve.MLX5DV_SIG_MASK_CRC32)
player.qp.wr_set_mkey_sig_block(sig_attr)
player.qp.wr_complete()
u.poll_cq(player.cq)
def traffic_with_bad_flow(client, server, iters, gid_idx, port):
"""
Runs basic traffic with bad flow between two sides
:param client: client side, clients base class is BaseTraffic
:param server: server side, servers base class is BaseTraffic
:param iters: number of traffic iterations
:param gid_idx: local gid index
:param port: IB port
:return: None
"""
import tests.utils as u
send_op = e.IBV_QP_EX_WITH_SEND
ah_client = u.get_global_ah(client, gid_idx, port)
s_recv_wr = u.get_recv_wr(server)
c_recv_wr = u.get_recv_wr(client)
for qp_idx in range(server.qp_count):
# Prepare the receive queue with RecvWR
u.post_recv(client, c_recv_wr, qp_idx=qp_idx)
u.post_recv(server, s_recv_wr, qp_idx=qp_idx)
read_offset = 0
for _ in range(iters):
for qp_idx in range(server.qp_count):
_, c_send_object = u.get_send_elements(client, False)
u.send(client, c_send_object, send_op, True, qp_idx, ah_client,
False)
try:
u.poll_cq(client.cq)
except PyverbsError as ex:
if client.bad_flow_handling(qp_idx, ex, True):
continue
raise ex
u.poll_cq(server.cq)
u.post_recv(server, s_recv_wr, qp_idx=qp_idx)
msg_received = server.mr.read(server.msg_size, read_offset)
u.validate(msg_received, True, server.msg_size)
client.check_after_traffic()
def invalidate_mw_type2_remote(self):
"""
Invalidate the MWs by sending invalidation send WR from the remote QP.
:return: None
"""
server_recv_wr = u.get_recv_wr(self.server)
client_recv_wr = u.get_recv_wr(self.client)
self.server.qp.post_recv(server_recv_wr)
self.client.qp.post_recv(client_recv_wr)
inv_send_wr = SendWR(opcode=e.IBV_WR_SEND_WITH_INV)
inv_send_wr.imm_data = self.client.rkey
self.client.qp.post_send(inv_send_wr)
inv_send_wr = SendWR(opcode=e.IBV_WR_SEND_WITH_INV)
inv_send_wr.imm_data = self.server.rkey
self.server.qp.post_send(inv_send_wr)
# Poll the invalidate MW send WR.
u.poll_cq(self.server.cq)
u.poll_cq(self.client.cq)
# Poll the invalidate MW recv WR.
u.poll_cq(self.server.cq)
u.poll_cq(self.client.cq)