def prepare_to_traffic(self):
"""
Prepare the TM SRQ for tag matching traffic by posing 33
(hardware limitation) recv WR for fill his queue
"""
for _ in range(self.server.qp_count):
u.post_recv(self.client, u.get_recv_wr(self.client), num_wqes=HW_LIMITAION)
u.post_recv(self.server, u.get_recv_wr(self.server), num_wqes=HW_LIMITAION)
def mixed_traffic(self):
s_recv_wr = u.get_recv_wr(self.server)
u.post_recv(self.server, s_recv_wr)
self.prepare_send_elements()
for i in range(self.iters):
self.client.qp.wr_start()
if i % 2:
self.client.mr.write('c' * self.client.mr.length,
self.client.mr.length)
self.client.qp.wr_flags = e.IBV_SEND_SIGNALED
self.client.qp.wr_send()
self.client.qp.wr_set_sge(self.regular_send_sge)
else:
self.client.mr.write('s' * self.client.mr.length,
self.client.mr.length)
self.client.qp.wr_raw_wqe(self.raw_send_wqe)
self.client.qp.wr_complete()
u.poll_cq_ex(self.client.cq)
u.poll_cq_ex(self.server.cq)
u.post_recv(self.server, s_recv_wr)
if not i % 2 and self.client.cq.read_opcode() != e.IBV_WC_DRIVER2:
raise PyverbsError(
'Opcode validation failed: expected '
f'{e.IBV_WC_DRIVER2}, received {self.client.cq.read_opcode()}'
)
act_buffer = self.server.mr.read(self.server.mr.length, 0)
u.validate(act_buffer, i % 2, self.server.mr.length)
def _ext_qp_server_traffic(self):
recv_wr = u.get_recv_wr(self.cm_res)
for _ in range(self.cm_res.num_msgs):
u.post_recv(self.cm_res, recv_wr)
self.syncer.wait()
for _ in range(self.cm_res.num_msgs):
u.poll_cq(self.cm_res.cq)
def _client_traffic_with_ext_qp(agr_obj, syncer):
recv_wr = get_recv_wr(agr_obj)
syncer.wait()
for _ in range(agr_obj.num_msgs):
send_wr = get_send_element(agr_obj, agr_obj.is_server)[0]
agr_obj.qp.post_send(send_wr)
poll_cq(agr_obj.cq)
agr_obj.qp.post_recv(recv_wr)
poll_cq(agr_obj.cq)
msg_received = agr_obj.mr.read(agr_obj.msg_size, 0)
validate(msg_received, agr_obj.is_server, agr_obj.msg_size)
def invalidate_mw_type2_remote(self):
"""
Invalidate the MWs by sending invalidation send WR from the remote QP.
:return: None
"""
server_recv_wr = u.get_recv_wr(self.server)
client_recv_wr = u.get_recv_wr(self.client)
self.server.qp.post_recv(server_recv_wr)
self.client.qp.post_recv(client_recv_wr)
inv_send_wr = SendWR(opcode=e.IBV_WR_SEND_WITH_INV)
inv_send_wr.imm_data = self.client.rkey
self.client.qp.post_send(inv_send_wr)
inv_send_wr = SendWR(opcode=e.IBV_WR_SEND_WITH_INV)
inv_send_wr.imm_data = self.server.rkey
self.server.qp.post_send(inv_send_wr)
# Poll the invalidate MW send WR.
u.poll_cq(self.server.cq)
u.poll_cq(self.client.cq)
# Poll the invalidate MW recv WR.
u.poll_cq(self.server.cq)
u.poll_cq(self.client.cq)
def ts_traffic(self):
"""
Run RDMA traffic and read the completions timestamps.
"""
s_recv_wr = u.get_recv_wr(self.server)
u.post_recv(self.server, s_recv_wr)
if self.qp_type == e.IBV_QPT_RAW_PACKET:
c_send_wr, _, _ = u.get_send_elements_raw_qp(self.client)
else:
c_send_wr, _ = u.get_send_elements(self.client, False)
u.send(self.client, c_send_wr, e.IBV_WR_SEND, False, 0)
self.client.timestamp = self.poll_cq_ex_ts(self.client.scq, ts_type=self.send_ts)
self.server.timestamp = self.poll_cq_ex_ts(self.server.rcq, ts_type=self.recv_ts)
def send_server_fdb_to_nic_packets(self, iters):
"""
Server sends and receives raw packets.
:param iters: Number of packets to send.
"""
s_recv_wr = u.get_recv_wr(self.server)
u.post_recv(self.server, s_recv_wr, qp_idx=0)
c_send_wr, _, msg = u.get_send_elements_raw_qp(self.server)
for _ in range(iters):
u.send(self.server, c_send_wr, e.IBV_WR_SEND)
u.poll_cq_ex(self.server.cq)
u.post_recv(self.server, s_recv_wr, qp_idx=0)
msg_received = self.server.mr.read(self.server.msg_size, 0)
u.validate_raw(msg_received, msg, [])
def traffic_with_bad_flow(client, server, iters, gid_idx, port):
"""
Runs basic traffic with bad flow between two sides
:param client: client side, clients base class is BaseTraffic
:param server: server side, servers base class is BaseTraffic
:param iters: number of traffic iterations
:param gid_idx: local gid index
:param port: IB port
:return: None
"""
import tests.utils as u
send_op = e.IBV_QP_EX_WITH_SEND
ah_client = u.get_global_ah(client, gid_idx, port)
s_recv_wr = u.get_recv_wr(server)
c_recv_wr = u.get_recv_wr(client)
for qp_idx in range(server.qp_count):
# Prepare the receive queue with RecvWR
u.post_recv(client, c_recv_wr, qp_idx=qp_idx)
u.post_recv(server, s_recv_wr, qp_idx=qp_idx)
read_offset = 0
for _ in range(iters):
for qp_idx in range(server.qp_count):
_, c_send_object = u.get_send_elements(client, False)
u.send(client, c_send_object, send_op, True, qp_idx, ah_client,
False)
try:
u.poll_cq(client.cq)
except PyverbsError as ex:
if client.bad_flow_handling(qp_idx, ex, True):
continue
raise ex
u.poll_cq(server.cq)
u.post_recv(server, s_recv_wr, qp_idx=qp_idx)
msg_received = server.mr.read(server.msg_size, read_offset)
u.validate(msg_received, True, server.msg_size)
client.check_after_traffic()
def test_mr_rereg_addr(self):
self.create_players(MRRes)
s_recv_wr = u.get_recv_wr(self.server)
self.server.qp.post_recv(s_recv_wr)
server_addr = posix_memalign(self.server.msg_size)
self.server.rereg_mr(flags=e.IBV_REREG_MR_CHANGE_TRANSLATION,
addr=server_addr,
length=self.server.msg_size)
with self.assertRaisesRegex(PyverbsRDMAError, 'Remote operation error'):
# The server QP receive queue has WR with the old MR address,
# therefore traffic should fail.
u.traffic(**self.traffic_args)
self.restate_qps()
u.traffic(**self.traffic_args)
free(server_addr)
def _ext_qp_client_traffic(self):
"""
RDMACM client side traffic function which sends and receives a message,
and then validates the received message. This traffic method uses the CM
external QP and CQ for send, recv and get_completion.
:return: None
"""
recv_wr = get_recv_wr(self.cm_res)
self.syncer.wait()
for _ in range(self.cm_res.num_msgs):
send_wr = get_send_elements(self.cm_res, self.cm_res.passive)[0]
self.cm_res.qp.post_send(send_wr)
poll_cq(self.cm_res.cq)
self.cm_res.qp.post_recv(recv_wr)
poll_cq(self.cm_res.cq)
msg_received = self.cm_res.mr.read(self.cm_res.msg_size, 0)
validate(msg_received, self.cm_res.passive, self.cm_res.msg_size)
def tm_traffic(self, tm_opcode=e.IBV_TMH_EAGER, fixed_send_tag=None):
"""
Runs Tag matching traffic between two sides (server and client)
:param tm_opcode: The TM opcode in the send WR
:param fixed_send_tag: If not None complitions are expected to be with no tag
"""
tags_list = list(range(1, self.iters))
for recv_tag in tags_list:
self.post_recv_tm(tag=recv_tag, wrid=recv_tag)
actual_cqe = self.poll_cq_ex(cqex=self.server.cq)
self.verify_cqe(actual_cqe=actual_cqe, wr_id=recv_tag, opcode=e.IBV_WC_TM_ADD)
tags_list.reverse()
for send_tag in tags_list:
send_tag, tag_exp, wrid_exp, wc_flags = self.get_exp_params(
fixed_send_tag=fixed_send_tag, send_tag=send_tag, tm_opcode=tm_opcode)
send_wr, exp_msg = self.get_send_elements(tag=send_tag, tm_opcode=tm_opcode)
u.send(self.client, send_wr)
self.poll_cq_ex(cqex=self.client.cq, to_valid=False)
actual_cqe = self.poll_cq_ex(cqex=self.server.cq)
exp_recv_tm_opcode = e.IBV_WC_TM_NO_TAG if tm_opcode == e.IBV_TMH_NO_TAG else \
e.IBV_WC_TM_RECV
self.verify_cqe(actual_cqe=actual_cqe, wr_id=wrid_exp, opcode=exp_recv_tm_opcode,
wc_flags=wc_flags, tag=tag_exp)
if tm_opcode == e.IBV_TMH_RNDV:
actual_cqe = self.poll_cq_ex(cqex=self.client.cq)
self.verify_cqe(actual_cqe=actual_cqe, opcode=e.IBV_WC_RECV, is_server=False)
actual_cqe = self.poll_cq_ex(cqex=self.server.cq)
self.verify_cqe(actual_cqe=actual_cqe, wr_id=wrid_exp, opcode=e.IBV_WC_TM_RECV,
wc_flags=e.IBV_WC_TM_DATA_VALID)
actual_msg, exp_msg, msg_size = self.build_expected_and_recv_msgs \
(exp_msg=exp_msg, tm_opcode=tm_opcode, fixed_send_tag=fixed_send_tag)
self.validate_msg(actual_msg, exp_msg, msg_size)
if fixed_send_tag and tm_opcode != e.IBV_TMH_NO_TAG:
self.validate_exp_recv_params(exp_parm=self.curr_unexpected_cnt,
recv_parm=self.server.unexp_cnt,
descriptor='unexpected_count')
self.curr_unexpected_cnt += 1
u.post_recv(self.server, u.get_recv_wr(self.server))
