def _ext_qp_server_traffic(self):
recv_wr = u.get_recv_wr(self.cm_res)
for _ in range(self.cm_res.num_msgs):
u.post_recv(self.cm_res, recv_wr)
self.syncer.wait()
for _ in range(self.cm_res.num_msgs):
u.poll_cq(self.cm_res.cq)
def mixed_traffic(self):
s_recv_wr = u.get_recv_wr(self.server)
u.post_recv(self.server, s_recv_wr)
self.prepare_send_elements()
for i in range(self.iters):
self.client.qp.wr_start()
if i % 2:
self.client.mr.write('c' * self.client.mr.length,
self.client.mr.length)
self.client.qp.wr_flags = e.IBV_SEND_SIGNALED
self.client.qp.wr_send()
self.client.qp.wr_set_sge(self.regular_send_sge)
else:
self.client.mr.write('s' * self.client.mr.length,
self.client.mr.length)
self.client.qp.wr_raw_wqe(self.raw_send_wqe)
self.client.qp.wr_complete()
u.poll_cq_ex(self.client.cq)
u.poll_cq_ex(self.server.cq)
u.post_recv(self.server, s_recv_wr)
if not i % 2 and self.client.cq.read_opcode() != e.IBV_WC_DRIVER2:
raise PyverbsError(
'Opcode validation failed: expected '
f'{e.IBV_WC_DRIVER2}, received {self.client.cq.read_opcode()}'
)
act_buffer = self.server.mr.read(self.server.mr.length, 0)
u.validate(act_buffer, i % 2, self.server.mr.length)
def get_send_elements(self, tag=0, tm_opcode=e.IBV_TMH_EAGER, tm=True):
"""
Creates a single SGE and a single Send WR for client QP. The content
of the message is 'c' for client side. The function also generates TMH
and RVH to the msg
:return: Send wr and expected msg that is read from mr
"""
sge = SGE(self.client.mr.buf, self.client.msg_size, self.client.mr_lkey)
if tm_opcode == e.IBV_TMH_RNDV:
max_rndv_hdr_size = self.server.ctx.query_device_ex().tm_caps.max_rndv_hdr_size
sge.length = max_rndv_hdr_size if max_rndv_hdr_size <= self.server.mr.length else \
self.server.mr.length
write_rndvu_header(player=self.client, mr=self.client.mr, tag=tag, tm_opcode=tm_opcode)
c_recv_wr = RecvWR(wr_id=tag, sg=[sge], num_sge=1)
# Need to post_recv client because the server sends rdma-read request to client
u.post_recv(self.client, c_recv_wr)
else:
msg = self.client.msg_size * 'c'
self.client.mr.write(msg, self.client.msg_size)
if tm:
write_tm_header(mr=self.client.mr, tag=tag, tm_opcode=tm_opcode)
send_wr = SendWR(opcode=e.IBV_WR_SEND, num_sge=1, sg=[sge])
exp_msg = self.client.mr.read(self.client.msg_size, 0)
return send_wr, exp_msg
def prepare_to_traffic(self):
"""
Prepare the TM SRQ for tag matching traffic by posing 33
(hardware limitation) recv WR for fill his queue
"""
for _ in range(self.server.qp_count):
u.post_recv(self.client, u.get_recv_wr(self.client), num_wqes=HW_LIMITAION)
u.post_recv(self.server, u.get_recv_wr(self.server), num_wqes=HW_LIMITAION)
def ts_traffic(self):
"""
Run RDMA traffic and read the completions timestamps.
"""
s_recv_wr = u.get_recv_wr(self.server)
u.post_recv(self.server, s_recv_wr)
if self.qp_type == e.IBV_QPT_RAW_PACKET:
c_send_wr, _, _ = u.get_send_elements_raw_qp(self.client)
else:
c_send_wr, _ = u.get_send_elements(self.client, False)
u.send(self.client, c_send_wr, e.IBV_WR_SEND, False, 0)
self.client.timestamp = self.poll_cq_ex_ts(self.client.scq, ts_type=self.send_ts)
self.server.timestamp = self.poll_cq_ex_ts(self.server.rcq, ts_type=self.recv_ts)
def send_server_fdb_to_nic_packets(self, iters):
"""
Server sends and receives raw packets.
:param iters: Number of packets to send.
"""
s_recv_wr = u.get_recv_wr(self.server)
u.post_recv(self.server, s_recv_wr, qp_idx=0)
c_send_wr, _, msg = u.get_send_elements_raw_qp(self.server)
for _ in range(iters):
u.send(self.server, c_send_wr, e.IBV_WR_SEND)
u.poll_cq_ex(self.server.cq)
u.post_recv(self.server, s_recv_wr, qp_idx=0)
msg_received = self.server.mr.read(self.server.msg_size, 0)
u.validate_raw(msg_received, msg, [])
def tm_traffic(self, tm_opcode=e.IBV_TMH_EAGER, fixed_send_tag=None):
"""
Runs Tag matching traffic between two sides (server and client)
:param tm_opcode: The TM opcode in the send WR
:param fixed_send_tag: If not None complitions are expected to be with no tag
"""
tags_list = list(range(1, self.iters))
for recv_tag in tags_list:
self.post_recv_tm(tag=recv_tag, wrid=recv_tag)
actual_cqe = self.poll_cq_ex(cqex=self.server.cq)
self.verify_cqe(actual_cqe=actual_cqe, wr_id=recv_tag, opcode=e.IBV_WC_TM_ADD)
tags_list.reverse()
for send_tag in tags_list:
send_tag, tag_exp, wrid_exp, wc_flags = self.get_exp_params(
fixed_send_tag=fixed_send_tag, send_tag=send_tag, tm_opcode=tm_opcode)
send_wr, exp_msg = self.get_send_elements(tag=send_tag, tm_opcode=tm_opcode)
u.send(self.client, send_wr)
self.poll_cq_ex(cqex=self.client.cq, to_valid=False)
actual_cqe = self.poll_cq_ex(cqex=self.server.cq)
exp_recv_tm_opcode = e.IBV_WC_TM_NO_TAG if tm_opcode == e.IBV_TMH_NO_TAG else \
e.IBV_WC_TM_RECV
self.verify_cqe(actual_cqe=actual_cqe, wr_id=wrid_exp, opcode=exp_recv_tm_opcode,
wc_flags=wc_flags, tag=tag_exp)
if tm_opcode == e.IBV_TMH_RNDV:
actual_cqe = self.poll_cq_ex(cqex=self.client.cq)
self.verify_cqe(actual_cqe=actual_cqe, opcode=e.IBV_WC_RECV, is_server=False)
actual_cqe = self.poll_cq_ex(cqex=self.server.cq)
self.verify_cqe(actual_cqe=actual_cqe, wr_id=wrid_exp, opcode=e.IBV_WC_TM_RECV,
wc_flags=e.IBV_WC_TM_DATA_VALID)
actual_msg, exp_msg, msg_size = self.build_expected_and_recv_msgs \
(exp_msg=exp_msg, tm_opcode=tm_opcode, fixed_send_tag=fixed_send_tag)
self.validate_msg(actual_msg, exp_msg, msg_size)
if fixed_send_tag and tm_opcode != e.IBV_TMH_NO_TAG:
self.validate_exp_recv_params(exp_parm=self.curr_unexpected_cnt,
recv_parm=self.server.unexp_cnt,
descriptor='unexpected_count')
self.curr_unexpected_cnt += 1
u.post_recv(self.server, u.get_recv_wr(self.server))
def traffic_with_bad_flow(client, server, iters, gid_idx, port):
"""
Runs basic traffic with bad flow between two sides
:param client: client side, clients base class is BaseTraffic
:param server: server side, servers base class is BaseTraffic
:param iters: number of traffic iterations
:param gid_idx: local gid index
:param port: IB port
:return: None
"""
import tests.utils as u
send_op = e.IBV_QP_EX_WITH_SEND
ah_client = u.get_global_ah(client, gid_idx, port)
s_recv_wr = u.get_recv_wr(server)
c_recv_wr = u.get_recv_wr(client)
for qp_idx in range(server.qp_count):
# Prepare the receive queue with RecvWR
u.post_recv(client, c_recv_wr, qp_idx=qp_idx)
u.post_recv(server, s_recv_wr, qp_idx=qp_idx)
read_offset = 0
for _ in range(iters):
for qp_idx in range(server.qp_count):
_, c_send_object = u.get_send_elements(client, False)
u.send(client, c_send_object, send_op, True, qp_idx, ah_client,
False)
try:
u.poll_cq(client.cq)
except PyverbsError as ex:
if client.bad_flow_handling(qp_idx, ex, True):
continue
raise ex
u.poll_cq(server.cq)
u.post_recv(server, s_recv_wr, qp_idx=qp_idx)
msg_received = server.mr.read(server.msg_size, read_offset)
u.validate(msg_received, True, server.msg_size)
client.check_after_traffic()