def build_traffic_elements_sig_pipelining(self):
"""
Build two WRs for data and response on client side and one WR for
response on server side. Transaction consists of two operations:
RDMA write of data and send/recv of response. Data size is
512 bytes, response size is 16 bytes. For simplicity the same
memory is used for data and response. Data is transferred using
the player signature mkey. Response is transferred using the
plain MR.
"""
server_sge_resp = SGE(self.server.mr.buf, 16, self.server.mr.lkey)
self.server_resp_wr = RecvWR(sg=[server_sge_resp], num_sge=1)
client_sge_data = SGE(0, 512, self.client.mkey.lkey)
self.client_data_wr = SendWR(wr_id=1,
opcode=e.IBV_WR_RDMA_WRITE,
num_sge=1,
sg=[client_sge_data],
send_flags=0)
self.client_data_wr.set_wr_rdma(self.server.mkey.rkey, 0)
client_sge_resp = SGE(self.client.mr.buf, 16, self.client.mr.lkey)
client_send_flags = e.IBV_SEND_SIGNALED | e.IBV_SEND_FENCE
self.client_resp_wr = SendWR(wr_id=1,
opcode=e.IBV_WR_SEND,
num_sge=1,
sg=[client_sge_resp],
send_flags=client_send_flags)
def get_send_elements(self, tag=0, tm_opcode=e.IBV_TMH_EAGER, tm=True):
"""
Creates a single SGE and a single Send WR for client QP. The content
of the message is 'c' for client side. The function also generates TMH
and RVH to the msg
:return: Send wr and expected msg that is read from mr
"""
sge = SGE(self.client.mr.buf, self.client.msg_size, self.client.mr_lkey)
if tm_opcode == e.IBV_TMH_RNDV:
max_rndv_hdr_size = self.server.ctx.query_device_ex().tm_caps.max_rndv_hdr_size
sge.length = max_rndv_hdr_size if max_rndv_hdr_size <= self.server.mr.length else \
self.server.mr.length
write_rndvu_header(player=self.client, mr=self.client.mr, tag=tag, tm_opcode=tm_opcode)
c_recv_wr = RecvWR(wr_id=tag, sg=[sge], num_sge=1)
# Need to post_recv client because the server sends rdma-read request to client
u.post_recv(self.client, c_recv_wr)
else:
msg = self.client.msg_size * 'c'
self.client.mr.write(msg, self.client.msg_size)
if tm:
write_tm_header(mr=self.client.mr, tag=tag, tm_opcode=tm_opcode)
send_wr = SendWR(opcode=e.IBV_WR_SEND, num_sge=1, sg=[sge])
exp_msg = self.client.mr.read(self.client.msg_size, 0)
return send_wr, exp_msg
def build_traffic_elements(self, sge_size):
"""
Build the server and client send/recv work requests.
:param sge_size: The sge send size using the mkey.
"""
opcode = e.IBV_WR_SEND
server_sge = SGE(0, sge_size, self.server.mkey.lkey)
self.server_recv_wr = RecvWR(sg=[server_sge], num_sge=1)
client_sge = SGE(0, sge_size, self.client.mkey.lkey)
self.client_send_wr = SendWR(opcode=opcode, num_sge=1, sg=[client_sge])
def get_recv_wr(agr_obj):
"""
Creates a single SGE Recv WR for agr_obj's QP type.
:param agr_obj: Aggregation object which contains all resources necessary
:return: recv wr
"""
qp_type = agr_obj.rqp_lst[0].qp_type if isinstance(agr_obj, XRCResources) \
else agr_obj.qp.qp_type
mr = agr_obj.mr
length = agr_obj.msg_size + GRH_SIZE if qp_type == e.IBV_QPT_UD \
else agr_obj.msg_size
recv_sge = SGE(mr.buf, length, mr.lkey)
return RecvWR(sg=[recv_sge], num_sge=1)
def get_recv_wr(agr_obj):
"""
Creates a single SGE Recv WR for agr_obj's QP type.
:param agr_obj: Aggregation object which contains all resources necessary
:return: recv wr
"""
qp_type = agr_obj.qp.qp_type
mr = agr_obj.mr
if qp_type == e.IBV_QPT_UD:
recv_sge = SGE(mr.buf, agr_obj.msg_size + GRH_SIZE, mr.lkey)
else:
recv_sge = SGE(mr.buf, agr_obj.msg_size, mr.lkey)
return RecvWR(sg=[recv_sge], num_sge=1)
def test_odp_ud_traffic(self):
client, server = self.create_players(OdpUD)
# Implement the traffic here because OdpUD uses two different MRs for
# send and recv.
recv_sge = SGE(server.recv_mr.buf, server.msg_size + GRH_SIZE,
server.recv_mr.lkey)
server_recv_wr = RecvWR(sg=[recv_sge], num_sge=1)
send_sge = SGE(client.send_mr.buf + GRH_SIZE, client.msg_size,
client.send_mr.lkey)
client_send_wr = SendWR(num_sge=1, sg=[send_sge])
for i in range(self.iters):
server.qp.post_recv(server_recv_wr)
post_send(client, client_send_wr, self.gid_index, self.ib_port)
poll_cq(client.cq)
poll_cq(server.cq)
def get_recv_wr(self, player, wire_encryption):
offset = 0 if wire_encryption else player.msg_size/2
sge = SGE(player.mr.buf + offset, player.msg_size/2, player.mr.lkey)
return RecvWR(sg=[sge], num_sge=1)