def rdma(self):
swr = ibv.send_wr(wr_id=0,
remote_addr=self.peerinfo.addr,
rkey=self.peerinfo.rkey,
sg_list=self.mr.sge(),
opcode=ibv.IBV_WR_RDMA_WRITE,
send_flags=ibv.IBV_SEND_SIGNALED)
n = self.opt.iters
depth = min(self.opt.tx_depth, n, self.qp.max_send_wr)
tpost = clock_monotonic()
for i in xrange(depth):
self.qp.post_send(swr)
completions = 0
posts = depth
for wc in self.poller.iterwc(timeout=1):
if wc.status != ibv.IBV_WC_SUCCESS:
raise ibv.WCError(wc,self.cq,obj=self.qp);
completions += 1
if posts < n:
self.qp.post_send(swr)
posts += 1
self.poller.wakeat = rdma.tools.clock_monotonic() + 1;
if completions == n:
break;
else:
raise rdma.RDMAError("CQ timed out");
tcomp = clock_monotonic()
rate = self.opt.size*self.opt.iters/1e6/(tcomp-tpost)
print "%.1f MB/sec" % rate
def make_send_wr(self,buf_idx,buf_len,path=None):
"""Return a :class:`rdma.ibverbs.send_wr` for *buf_idx* and path.
If *path* is `None` then the wr does not contain path information
(eg for connected QPs)"""
if path is not None:
return ibv.send_wr(wr_id=buf_idx,
sg_list=self.make_sge(buf_idx,buf_len),
opcode=ibv.IBV_WR_SEND,
send_flags=ibv.IBV_SEND_SIGNALED,
ah=self._mr.pd.ah(path),
remote_qpn=path.dqpn,
remote_qkey=path.qkey);
else:
return ibv.send_wr(wr_id=buf_idx,
sg_list=self.make_sge(buf_idx,buf_len),
opcode=ibv.IBV_WR_SEND,
send_flags=ibv.IBV_SEND_SIGNALED);
def make_send_wr(self, buf_idx, buf_len, path=None):
"""Return a :class:`rdma.ibverbs.send_wr` for *buf_idx* and path.
If *path* is `None` then the wr does not contain path information
(eg for connected QPs)"""
if path is not None:
return ibv.send_wr(wr_id=buf_idx,
sg_list=self.make_sge(buf_idx, buf_len),
opcode=ibv.IBV_WR_SEND,
send_flags=ibv.IBV_SEND_SIGNALED,
ah=self._mr.pd.ah(path),
remote_qpn=path.dqpn,
remote_qkey=path.qkey)
else:
return ibv.send_wr(wr_id=buf_idx,
sg_list=self.make_sge(buf_idx, buf_len),
opcode=ibv.IBV_WR_SEND,
send_flags=ibv.IBV_SEND_SIGNALED)
def rdma(self):
if self.opt.num_sge > 1:
block = self.opt.size / self.opt.num_sge + 1
sg_list = []
offset = 0
while offset < self.opt.size:
if offset + block > self.opt.size:
block = self.opt.size - offset
sg_list.append( self.mr.sge( block, offset ) )
offset += block
else:
sg_list = self.mr.sge()
swr = ibv.send_wr(wr_id=0,
remote_addr=self.peerinfo[2],
sg_list=sg_list,
rkey=self.peerinfo[3],
opcode=ibv.IBV_WR_RDMA_WRITE,
send_flags=ibv.IBV_SEND_SIGNALED)
n = self.opt.iters
depth = min(self.opt.tx_depth, n, self.qp.max_send_wr)
tpost = clock_monotonic()
for i in xrange(depth):
self.qp.post_send(swr)
completions = 0
posts = depth
for wc in self.poller.iterwc(timeout=100):
if wc.status != ibv.IBV_WC_SUCCESS:
raise ibv.WCError(wc,self.cq,obj=self.qp);
completions += 1
if posts < n:
self.qp.post_send(swr)
posts += 1
self.poller.wakeat = rdma.tools.clock_monotonic() + 1;
if completions == n:
break;
else:
raise rdma.RDMAError("CQ timed out");
tcomp = clock_monotonic()
rate = self.opt.size*self.opt.iters/1e6/(tcomp-tpost)
print "%.1f MB/sec" % rate