def test_create_qp_with_attr_connected(self): """ Test QP creation via ibv_create_qp without a QPAttr object proivded. QP type can be either RC or UC. """ for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: with CQ(ctx, 100, None, None, 0) as cq: qia = get_qp_init_attr(cq, [e.IBV_QPT_RC, e.IBV_QPT_UC], attr) with QP(pd, qia, QPAttr()) as qp: assert qp.qp_state == e.IBV_QPS_INIT
def test_create_qp_ex_with_attr(self): """ Test QP creation via ibv_create_qp_ex with a QPAttr object proivded. Checked QP types are Raw Packet and UD. Raw Packet is skipped for non-root users / Infiniband link layer. """ for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: with CQ(ctx, 100, None, None, 0) as cq: for i in range(1, attr.phys_port_cnt + 1): qpts = [e.IBV_QPT_UD, e.IBV_QPT_RAW_PACKET] \ if is_eth(ctx, i) else [e.IBV_QPT_UD] qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex, e.IBV_QPT_UD) with QP(ctx, qia, QPAttr()) as qp: assert qp.qp_state == e.IBV_QPS_RTS, 'UD QP should have been in RTS' if is_eth(ctx, i) and is_root(): qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex, e.IBV_QPT_RAW_PACKET) with QP(ctx, qia, QPAttr()) as qp: assert qp.qp_state == e.IBV_QPS_RTS, 'Raw Packet QP should have been in RTS'
def create_qp(self): """ Initializes self.qp with an RC QP. :return: None """ qp_caps = QPCap(max_recv_wr=self.num_msgs) qp_init_attr = QPInitAttr(qp_type=e.IBV_QPT_RC, scq=self.cq, rcq=self.cq, cap=qp_caps) qp_attr = QPAttr(port_num=self.ib_port) self.qp = QP(self.pd, qp_init_attr, qp_attr)
def test_modify_ud_qp(self): """ Queries a UD QP after calling modify(). Verifies that its properties are as expected. """ with PD(self.ctx) as pd: with CQ(self.ctx, 100, None, None, 0) as cq: # Legacy QP qia = u.get_qp_init_attr(cq, self.attr) qia.qp_type = e.IBV_QPT_UD qp = self.create_qp(pd, qia, False, False, self.ib_port) qa = QPAttr() qa.qkey = 0x123 qp.to_init(qa) qp_attr, _ = qp.query(e.IBV_QP_QKEY) assert qp_attr.qkey == qa.qkey, 'Legacy QP, QKey is not as expected' qp.to_rtr(qa) qa.sq_psn = 0x45 qp.to_rts(qa) qp_attr, _ = qp.query(e.IBV_QP_SQ_PSN) assert qp_attr.sq_psn == qa.sq_psn, 'Legacy QP, SQ PSN is not as expected' qa.qp_state = e.IBV_QPS_RESET qp.modify(qa, e.IBV_QP_STATE) assert qp.qp_state == e.IBV_QPS_RESET, 'Legacy QP, QP state is not as expected' # Extended QP qia = get_qp_init_attr_ex(cq, pd, self.attr, self.attr_ex, e.IBV_QPT_UD) qp = self.create_qp(self.ctx, qia, True, False, self.ib_port) qa = QPAttr() qa.qkey = 0x123 qp.to_init(qa) qp_attr, _ = qp.query(e.IBV_QP_QKEY) assert qp_attr.qkey == qa.qkey, 'Extended QP, QKey is not as expected' qp.to_rtr(qa) qa.sq_psn = 0x45 qp.to_rts(qa) qp_attr, _ = qp.query(e.IBV_QP_SQ_PSN) assert qp_attr.sq_psn == qa.sq_psn, 'Extended QP, SQ PSN is not as expected' qa.qp_state = e.IBV_QPS_RESET qp.modify(qa, e.IBV_QP_STATE) assert qp.qp_state == e.IBV_QPS_RESET, 'Extended QP, QP state is not as expected'
def create_qp(self, creator, qp_init_attr, is_ex, with_attr, port_num): """ Auxiliary function to create QP object. """ try: qp_attr = (None, QPAttr(port_num=port_num))[with_attr] return QP(creator, qp_init_attr, qp_attr) except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: with_str = ('without', 'with')[with_attr] + ('', ' extended')[is_ex] qp_type_str = pu.qp_type_to_str(qp_init_attr.qp_type) raise unittest.SkipTest(f'Create {qp_type_str} QP {with_str} attrs is not supported') raise ex
def reset_qp(self, qp_idx): qp_attr = QPAttr(qp_state=e.IBV_QPS_RESET) self.qps[qp_idx].modify(qp_attr, e.IBV_QP_STATE) self.qps[qp_idx].to_rts(qp_attr) err_mask = self.qp_stream_errors[qp_idx][0] index = 0 # Clear all set dci while err_mask != 0: if (err_mask & 0x1) == 0x1: Mlx5QP.modify_dci_stream_channel_id(self.qps[qp_idx], index) index += 1 err_mask >>= 1 self.qp_stream_errors[qp_idx][0] = 0
def create_qp(self): """ Create an rdmacm QP. If self.with_ext_qp is set, then an external CQ and RC QP will be created and set in self.cq and self.qp respectively. """ cmid = self.child_id if self.passive else self.cmid if not self.with_ext_qp: cmid.create_qp(self.create_qp_init_attr()) else: self.cq = CQ(cmid.context, self.num_msgs, None, None, 0) init_attr = self.create_qp_init_attr(rcq=self.cq, scq=self.cq) self.qp = QP(cmid.pd, init_attr, QPAttr())
def test_create_qp_with_attr(self): """ Test QP creation via ibv_create_qp with a QPAttr object proivded. QP type can be either Raw Packet or UD. """ for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: with CQ(ctx, 100, None, None, 0) as cq: for i in range(1, attr.phys_port_cnt + 1): qpts = [e.IBV_QPT_UD, e.IBV_QPT_RAW_PACKET] \ if is_eth(ctx, i) else [e.IBV_QPT_UD] qia = get_qp_init_attr(cq, qpts, attr) with QP(pd, qia, QPAttr()) as qp: assert qp.qp_state == e.IBV_QPS_RTS
def create_qp(self, conn_idx=0): """ Create an rdmacm QP. If self.with_ext_qp is set, then an external CQ and QP will be created. In case that CQ is already created, it is used for the newly created QP. :param conn_idx: The connection index. """ cmid = self.child_id if self.passive else self.cmid if not self.with_ext_qp: cmid.create_qp(self.create_qp_init_attr()) else: self.create_cq(cmid) init_attr = self.create_qp_init_attr(rcq=self.cq, scq=self.cq) self.qps[conn_idx] = QP(cmid.pd, init_attr, QPAttr())
def create_qp(self): qia = QPInitAttr(e.IBV_QPT_RAW_PACKET, rcq=self.cq, scq=self.cq, cap=QPCap()) try: qp = QP(self.pd, qia) except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest( "Create Raw Packet QP is not supported") raise ex qp.to_init(QPAttr()) return qp
def to_rts(self): gid = self.ctx.query_gid(self.ib_port, self.gid_index) gr = GlobalRoute(dgid=gid, sgid_index=self.gid_index) ah_attr = AHAttr(port_num=self.ib_port, is_global=True, gr=gr, dlid=self.port_attr.lid) qp_attr = QPAttr() qp_attr.path_mtu = PATH_MTU set_rnr_attributes(qp_attr) qp_attr.ah_attr = ah_attr for i in range(self.qp_count): qp_attr.dest_qp_num = self.rqps_num[i][1] qp_attr.rq_psn = self.psns[i] qp_attr.sq_psn = self.rpsns[i] self.rqp_lst[i].to_rts(qp_attr) qp_attr.dest_qp_num = self.rqps_num[i][0] self.sqp_lst[i].to_rts(qp_attr)
def create_qp_ex(agr_obj, qp_type, send_flags): if qp_type == e.IBV_QPT_XRC_SEND: cap = QPCap(max_send_wr=agr_obj.num_msgs, max_recv_wr=0, max_recv_sge=0, max_send_sge=1) else: cap = QPCap(max_send_wr=agr_obj.num_msgs, max_recv_wr=agr_obj.num_msgs, max_recv_sge=1, max_send_sge=1) qia = QPInitAttrEx(cap=cap, qp_type=qp_type, scq=agr_obj.cq, rcq=agr_obj.cq, pd=agr_obj.pd, send_ops_flags=send_flags, comp_mask=e.IBV_QP_INIT_ATTR_PD | e.IBV_QP_INIT_ATTR_SEND_OPS_FLAGS) qp_attr = QPAttr(port_num=agr_obj.ib_port) if qp_type == e.IBV_QPT_UD: qp_attr.qkey = agr_obj.UD_QKEY qp_attr.pkey_index = agr_obj.UD_PKEY_INDEX if qp_type == e.IBV_QPT_RC: qp_attr.qp_access_flags = e.IBV_ACCESS_REMOTE_WRITE | \ e.IBV_ACCESS_REMOTE_READ | \ e.IBV_ACCESS_REMOTE_ATOMIC try: # We don't have capability bits for this qp = QPEx(agr_obj.ctx, qia, qp_attr) except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest( 'Extended QP is not supported on this device') raise ex return qp
def test_mkey_sig_pipelining_bad(self): """ Test the bad signature pipelining scenario. """ self.create_players( Mlx5MkeyResources, dv_send_ops_flags=dve.MLX5DV_QP_EX_WITH_MKEY_CONFIGURE, mkey_create_flags=dve.MLX5DV_MKEY_INIT_ATTR_FLAGS_INDIRECT | dve.MLX5DV_MKEY_INIT_ATTR_FLAGS_BLOCK_SIGNATURE, dv_qp_create_flags=dve.MLX5DV_QP_CREATE_DISABLE_SCATTER_TO_CQE | dve.MLX5DV_QP_CREATE_SIG_PIPELINING) self.reg_mr_sig_pipelining_client(check_mask=dve.MLX5DV_SIG_MASK_CRC32) self.reg_mr_sig_pipelining_server() self.build_traffic_elements_sig_pipelining() self.server.qp.post_recv(self.server_resp_wr) self.client.qp.post_send(self.client_data_wr) self.client.qp.post_send(self.client_resp_wr) # Expect SQ_DRAINED event event = self.client.ctx.get_async_event() event.ack() self.assertEqual(event.event_type, e.IBV_EVENT_SQ_DRAINED) # No completion is expected on the client side nc, _ = self.client.cq.poll(1) self.assertEqual(nc, 0) # No completion is expected on the server side nc, _ = self.server.cq.poll(1) self.assertEqual(nc, 0) self.check_mkey(self.client, dve.MLX5DV_MKEY_SIG_BLOCK_BAD_GUARD) self.check_mkey(self.server) # Cancel and repost response WR canceled_count = self.client.qp.cancel_posted_send_wrs(1) self.assertEqual(canceled_count, 1) self.client.qp.post_send(self.client_resp_wr) # Move QP back to RTS and receive completions self.client.qp.modify( QPAttr(qp_state=e.IBV_QPS_RTS, cur_qp_state=e.IBV_QPS_SQD), e.IBV_QP_STATE | e.IBV_QP_CUR_STATE) u.poll_cq(self.client.cq) u.poll_cq(self.server.cq)
def create_qp(self): qia = QPInitAttr(e.IBV_QPT_RAW_PACKET, rcq=self.cq, scq=self.cq, cap=QPCap()) qp = QP(self.pd, qia) qp.to_init(QPAttr()) return qp
def to_rts(self): gid = self.ctx.query_gid(self.ib_port, self.gid_index) gr = GlobalRoute(dgid=gid, sgid_index=self.gid_index) ah_attr = AHAttr(port_num=self.ib_port, is_global=True, gr=gr, dlid=self.port_attr.lid) qp_attr = QPAttr() qp_attr.path_mtu = PATH_MTU qp_attr.timeout = TIMEOUT qp_attr.retry_cnt = RETRY_CNT qp_attr.rnr_retry = RNR_RETRY qp_attr.min_rnr_timer = MIN_RNR_TIMER qp_attr.ah_attr = ah_attr for i in range(self.qp_count): qp_attr.dest_qp_num = self.rqps_num[i][1] qp_attr.rq_psn = self.psns[i] qp_attr.sq_psn = self.rpsns[i] self.rqp_lst[i].to_rts(qp_attr) qp_attr.dest_qp_num = self.rqps_num[i][0] self.sqp_lst[i].to_rts(qp_attr)
def to_rts(self): """ Set the QP attributes' values to arbitrary values (same values used in ibv_rc_pingpong). :return: None """ attr = QPAttr(port_num=self.ib_port) attr.dest_qp_num = self.rqpn attr.path_mtu = PATH_MTU attr.max_dest_rd_atomic = MAX_DEST_RD_ATOMIC attr.min_rnr_timer = MIN_RNR_TIMER attr.rq_psn = self.psn attr.sq_psn = self.rpsn attr.timeout = TIMEOUT attr.retry_cnt = RETRY_CNT attr.rnr_retry = RNR_RETRY attr.max_rd_atomic = MAX_RD_ATOMIC gr = GlobalRoute(dgid=self.ctx.query_gid(self.ib_port, self.gid_index), sgid_index=self.gid_index) ah_attr = AHAttr(port_num=self.ib_port, is_global=1, gr=gr, dlid=self.port_attr.lid) attr.ah_attr = ah_attr self.qp.to_rts(attr)
def create_qp_attr(self): qp_attr = QPAttr(port_num=self.ib_port) qp_attr.qp_access_flags = self.qp_access return qp_attr
def create_qp_attr(self): attr = QPAttr(port_num=self.ib_port) attr.qkey = self.SRD_QKEY attr.pkey_index = self.SRD_PKEY_INDEX return attr
def create_qp_attr(self): return QPAttr(port_num=self.ib_port)
def create_qp_attr(self): qp_attr = QPAttr(port_num=self.ib_port) qp_attr.qp_access_flags = e.IBV_ACCESS_LOCAL_WRITE if self.remote_access: qp_attr.qp_access_flags |= e.IBV_ACCESS_REMOTE_WRITE return qp_attr
def create_qp_attr(self): qp_attr = QPAttr(port_num=self.ib_port) qp_access = e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_REMOTE_WRITE qp_attr.qp_access_flags = qp_access return qp_attr
def reset_qp(self, qp_idx): qp_attr = QPAttr(qp_state=e.IBV_QPS_RESET) self.qps[qp_idx].modify(qp_attr, e.IBV_QP_STATE) self.qps[qp_idx].to_rts(qp_attr) self.qp_stream_errors[qp_idx][0] = 0