def rdma(self): swr = ibv.send_wr(wr_id=0, remote_addr=self.peerinfo.addr, rkey=self.peerinfo.rkey, sg_list=self.mr.sge(), opcode=ibv.IBV_WR_RDMA_WRITE, send_flags=ibv.IBV_SEND_SIGNALED) n = self.opt.iters depth = min(self.opt.tx_depth, n, self.qp.max_send_wr) tpost = clock_monotonic() for i in xrange(depth): self.qp.post_send(swr) completions = 0 posts = depth for wc in self.poller.iterwc(timeout=1): if wc.status != ibv.IBV_WC_SUCCESS: raise ibv.WCError(wc,self.cq,obj=self.qp); completions += 1 if posts < n: self.qp.post_send(swr) posts += 1 self.poller.wakeat = rdma.tools.clock_monotonic() + 1; if completions == n: break; else: raise rdma.RDMAError("CQ timed out"); tcomp = clock_monotonic() rate = self.opt.size*self.opt.iters/1e6/(tcomp-tpost) print "%.1f MB/sec" % rate
def rdma(self): if self.opt.num_sge > 1: block = self.opt.size / self.opt.num_sge + 1 sg_list = [] offset = 0 while offset < self.opt.size: if offset + block > self.opt.size: block = self.opt.size - offset sg_list.append( self.mr.sge( block, offset ) ) offset += block else: sg_list = self.mr.sge() swr = ibv.send_wr(wr_id=0, remote_addr=self.peerinfo[2], sg_list=sg_list, rkey=self.peerinfo[3], opcode=ibv.IBV_WR_RDMA_WRITE, send_flags=ibv.IBV_SEND_SIGNALED) n = self.opt.iters depth = min(self.opt.tx_depth, n, self.qp.max_send_wr) tpost = clock_monotonic() for i in xrange(depth): self.qp.post_send(swr) completions = 0 posts = depth for wc in self.poller.iterwc(timeout=100): if wc.status != ibv.IBV_WC_SUCCESS: raise ibv.WCError(wc,self.cq,obj=self.qp); completions += 1 if posts < n: self.qp.post_send(swr) posts += 1 self.poller.wakeat = rdma.tools.clock_monotonic() + 1; if completions == n: break; else: raise rdma.RDMAError("CQ timed out"); tcomp = clock_monotonic() rate = self.opt.size*self.opt.iters/1e6/(tcomp-tpost) print "%.1f MB/sec" % rate