def _ext_qp_client_traffic(self): self.cm_res.remote_dct_num = self.cm_res.remote_qpn _, send_wr = u.get_send_elements(self.cm_res, self.cm_res.passive) ah = AH(self.cm_res.cmid.pd, attr=self.cm_res.remote_ah) self.syncer.wait() for send_idx in range(self.cm_res.num_msgs): dci_idx = send_idx % len(self.cm_res.qps) u.post_send_ex(self.cm_res, send_wr, e.IBV_QP_EX_WITH_SEND, ah=ah, qp_idx=dci_idx) u.poll_cq(self.cm_res.cq)
def ts_traffic(self): """ Run RDMA traffic and read the completions timestamps. """ s_recv_wr = u.get_recv_wr(self.server) u.post_recv(self.server, s_recv_wr) if self.qp_type == e.IBV_QPT_RAW_PACKET: c_send_wr, _, _ = u.get_send_elements_raw_qp(self.client) else: c_send_wr, _ = u.get_send_elements(self.client, False) u.send(self.client, c_send_wr, e.IBV_WR_SEND, False, 0) self.client.timestamp = self.poll_cq_ex_ts(self.client.scq, ts_type=self.send_ts) self.server.timestamp = self.poll_cq_ex_ts(self.server.rcq, ts_type=self.recv_ts)
def _ext_qp_client_traffic(self): """ RDMACM client side traffic function which sends and receives a message, and then validates the received message. This traffic method uses the CM external QP and CQ for send, recv and get_completion. :return: None """ recv_wr = get_recv_wr(self.cm_res) self.syncer.wait() for _ in range(self.cm_res.num_msgs): send_wr = get_send_elements(self.cm_res, self.cm_res.passive)[0] self.cm_res.qp.post_send(send_wr) poll_cq(self.cm_res.cq) self.cm_res.qp.post_recv(recv_wr) poll_cq(self.cm_res.cq) msg_received = self.cm_res.mr.read(self.cm_res.msg_size, 0) validate(msg_received, self.cm_res.passive, self.cm_res.msg_size)
def test_qp_ex_rc_bind_mw(self): """ Verify bind memory window operation using the new post_send API. Instead of checking through regular pingpong style traffic, we'll do as follows: - Register an MR with remote write access - Bind a MW without remote write permission to the MR - Verify that remote write fails Since it's a unique flow, it's an integral part of that test rather than a utility method. """ client, server = self.create_players('rc_bind_mw') client_sge = u.get_send_elements(client, False)[1] # Create a MW and bind it server.qp.wr_start() server.qp.wr_id = 0x123 server.qp.wr_flags = e.IBV_SEND_SIGNALED bind_info = MWBindInfo(server.mr, server.mr.buf, server.mr.length, e.IBV_ACCESS_LOCAL_WRITE) try: mw = MW(server.pd, mw_type=e.IBV_MW_TYPE_2) except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest( 'Memory Window allocation is not supported') raise ex new_key = inc_rkey(server.mr.rkey) server.qp.wr_bind_mw(mw, new_key, bind_info) server.qp.wr_complete() u.poll_cq(server.cq) # Verify that remote write fails client.qp.wr_start() client.qp.wr_id = 0x124 client.qp.wr_flags = e.IBV_SEND_SIGNALED client.qp.wr_rdma_write(new_key, server.mr.buf) client.qp.wr_set_sge(client_sge) client.qp.wr_complete() try: u.poll_cq(client.cq) except PyverbsRDMAError as ex: if ex.error_code != e.IBV_WC_REM_ACCESS_ERR: raise ex
def full_sq_bad_flow(self): """ Check post_send while qp's sq is full. - Find qp's sq length - Fill the qp with work requests until overflow """ qp_idx = 0 send_op = e.IBV_QP_EX_WITH_SEND ah = u.get_global_ah(self.client, self.gid_index, self.ib_port) qp_attr, _ = self.client.qps[qp_idx].query(e.IBV_QP_CAP) max_send_wr = qp_attr.cap.max_send_wr with self.assertRaises(PyverbsRDMAError) as ex: for _ in range(max_send_wr + 1): _, c_sg = u.get_send_elements(self.client, False) u.send(self.client, c_sg, send_op, new_send=True, qp_idx=qp_idx, ah=ah) self.assertEqual(ex.exception.error_code, errno.ENOMEM)
def test_resize_cq(self): """ Test resize CQ, start with specific value and then increase and decrease the CQ size. The test also check bad flow of decrease the CQ size when there are more completions on it than the new value. """ self.create_players(CQUDResources, cq_depth=3) # Decrease the CQ size. new_cq_size = 1 try: self.client.cq.resize(new_cq_size) except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest('Resize CQ is not supported') raise ex self.assertTrue( self.client.cq.cqe >= new_cq_size, f'The actual CQ size ({self.client.cq.cqe}) is less ' 'than guaranteed ({new_cq_size})') # Increase the CQ size. new_cq_size = 7 self.client.cq.resize(new_cq_size) self.assertTrue( self.client.cq.cqe >= new_cq_size, f'The actual CQ size ({self.client.cq.cqe}) is less ' 'than guaranteed ({new_cq_size})') # Fill the CQ entries except one for avoid cq_overrun warnings. send_wr, _ = u.get_send_elements(self.client, False) ah_client = u.get_global_ah(self.client, self.gid_index, self.ib_port) for i in range(self.client.cq.cqe - 1): u.send(self.client, send_wr, ah=ah_client) # Decrease the CQ size to less than the CQ unpolled entries. new_cq_size = 1 with self.assertRaises(PyverbsRDMAError) as ex: self.client.cq.resize(new_cq_size) self.assertEqual(ex.exception.error_code, errno.EINVAL)
def traffic_with_bad_flow(client, server, iters, gid_idx, port): """ Runs basic traffic with bad flow between two sides :param client: client side, clients base class is BaseTraffic :param server: server side, servers base class is BaseTraffic :param iters: number of traffic iterations :param gid_idx: local gid index :param port: IB port :return: None """ import tests.utils as u send_op = e.IBV_QP_EX_WITH_SEND ah_client = u.get_global_ah(client, gid_idx, port) s_recv_wr = u.get_recv_wr(server) c_recv_wr = u.get_recv_wr(client) for qp_idx in range(server.qp_count): # Prepare the receive queue with RecvWR u.post_recv(client, c_recv_wr, qp_idx=qp_idx) u.post_recv(server, s_recv_wr, qp_idx=qp_idx) read_offset = 0 for _ in range(iters): for qp_idx in range(server.qp_count): _, c_send_object = u.get_send_elements(client, False) u.send(client, c_send_object, send_op, True, qp_idx, ah_client, False) try: u.poll_cq(client.cq) except PyverbsError as ex: if client.bad_flow_handling(qp_idx, ex, True): continue raise ex u.poll_cq(server.cq) u.post_recv(server, s_recv_wr, qp_idx=qp_idx) msg_received = server.mr.read(server.msg_size, read_offset) u.validate(msg_received, True, server.msg_size) client.check_after_traffic()