def test_create_qp_no_attr(self): """ Test QP creation via ibv_create_qp without a QPAttr object proivded. Checked QP types are Raw Packet and UD. Raw Packet is skipped for non-root users / Infiniband link layer. """ for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: with CQ(ctx, 100, None, None, 0) as cq: for i in range(1, attr.phys_port_cnt + 1): qia = get_qp_init_attr(cq, attr) qia.qp_type = e.IBV_QPT_UD with QP(pd, qia) as qp: assert qp.qp_state == e.IBV_QPS_RESET, 'UD QP should have been in RESET' if u.is_eth(ctx, i) and u.is_root(): qia.qp_type = e.IBV_QPT_RAW_PACKET try: with QP(pd, qia) as qp: assert qp.qp_state == e.IBV_QPS_RESET, 'Raw Packet QP should have been in RESET' except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest( "Create Raw Packet QP is not supported" ) raise ex
def test_create_qp_no_attr_connected(self): """ Test QP creation via ibv_create_qp without a QPAttr object proivded. Checked QP types are RC and UC. """ for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: with CQ(ctx, 100, None, None, 0) as cq: qia = get_qp_init_attr(cq, attr) qia.qp_type = e.IBV_QPT_RC try: with QP(pd, qia) as qp: assert qp.qp_state == e.IBV_QPS_RESET, 'RC QP should have been in RESET' except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest( 'Create QP with RC attrs is not supported') raise ex qia.qp_type = e.IBV_QPT_UC try: with QP(pd, qia) as qp: assert qp.qp_state == e.IBV_QPS_RESET, 'UC QP should have been in RESET' except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest( 'Create QP with UC attrs is not supported') raise ex
def create_qps(self): """ Initializes self.qps with an XRC SEND/RECV QPs. :return: None """ qp_attr = QPAttr(port_num=self.ib_port) qp_attr.pkey_index = 0 for _ in range(self.qp_count): attr_ex = QPInitAttrEx(qp_type=e.IBV_QPT_XRC_RECV, comp_mask=e.IBV_QP_INIT_ATTR_XRCD, xrcd=self.xrcd) qp_attr.qp_access_flags = e.IBV_ACCESS_REMOTE_WRITE | \ e.IBV_ACCESS_REMOTE_READ recv_qp = QP(self.ctx, attr_ex, qp_attr) self.rqp_lst.append(recv_qp) qp_caps = QPCap(max_send_wr=self.num_msgs, max_recv_sge=0, max_recv_wr=0) attr_ex = QPInitAttrEx(qp_type=e.IBV_QPT_XRC_SEND, sq_sig_all=1, comp_mask=e.IBV_QP_INIT_ATTR_PD, pd=self.pd, scq=self.cq, cap=qp_caps) qp_attr.qp_access_flags = 0 send_qp =QP(self.ctx, attr_ex, qp_attr) self.sqp_lst.append(send_qp) self.qps_num.append((recv_qp.qp_num, send_qp.qp_num)) self.psns.append(random.getrandbits(24))
def test_create_qp_ex_with_attr(self): """ Test QP creation via ibv_create_qp_ex with a QPAttr object proivded. Checked QP types are Raw Packet and UD. Raw Packet is skipped for non-root users / Infiniband link layer. """ for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: with CQ(ctx, 100, None, None, 0) as cq: for i in range(1, attr.phys_port_cnt + 1): qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex, e.IBV_QPT_UD) try: with QP(ctx, qia, QPAttr()) as qp: assert qp.qp_state == e.IBV_QPS_RTS, 'UD QP should have been in RTS' except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest('Create QP with extended attrs is not supported') raise ex if is_eth(ctx, i) and is_root(): qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex, e.IBV_QPT_RAW_PACKET) try: with QP(ctx, qia, QPAttr()) as qp: assert qp.qp_state == e.IBV_QPS_RTS, 'Raw Packet QP should have been in RTS' except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest('Create QP with extended attrs is not supported') raise ex
def test_create_qp_ex_with_attr_connected(self): """ Test QP creation via ibv_create_qp_ex with a QPAttr object proivded. Checked QP type are RC and UC. """ for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: with CQ(ctx, 100, None, None, 0) as cq: qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex, e.IBV_QPT_RC) try: with QP(ctx, qia, QPAttr()) as qp: assert qp.qp_state == e.IBV_QPS_INIT, 'RC QP should have been in INIT' except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest('Create QP with extended attrs is not supported') raise ex qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex, e.IBV_QPT_UC) try: with QP(ctx, qia, QPAttr()) as qp: assert qp.qp_state == e.IBV_QPS_INIT, 'UC QP should have been in INIT' except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest('Create QP with extended attrs is not supported') raise ex
def create_qp(self): """ Create an rdmacm QP. If self.with_ext_qp is set, then an external CQ and RC QP will be created and set in self.cq and self.qp respectively. """ cmid = self.child_id if self.passive else self.cmid if not self.with_ext_qp: cmid.create_qp(self.create_qp_init_attr()) else: self.cq = CQ(cmid.context, self.num_msgs, None, None, 0) init_attr = self.create_qp_init_attr(rcq=self.cq, scq=self.cq) self.qp = QP(cmid.pd, init_attr, QPAttr())
def create_qp(self): qia = QPInitAttr(e.IBV_QPT_RAW_PACKET, rcq=self.cq, scq=self.cq, cap=QPCap()) try: qp = QP(self.pd, qia) except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest( "Create Raw Packet QP is not supported") raise ex qp.to_init(QPAttr()) return qp
def test_create_qp_with_attr_connected(self): """ Test QP creation via ibv_create_qp without a QPAttr object proivded. Checked QP types are RC and UC. """ for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: with CQ(ctx, 100, None, None, 0) as cq: qia = get_qp_init_attr(cq, attr) qia.qp_type = e.IBV_QPT_RC with QP(pd, qia, QPAttr()) as qp: assert qp.qp_state == e.IBV_QPS_INIT, 'RC QP should have been in INIT' qia.qp_type = e.IBV_QPT_UC with QP(pd, qia, QPAttr()) as qp: assert qp.qp_state == e.IBV_QPS_INIT, 'UC QP should have been in INIT'
def test_create_qp_ex_no_attr_connected(self): """ Test QP creation via ibv_create_qp_ex without a QPAttr object proivded. Checked QP types are RC and UC. """ for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: with CQ(ctx, 100, None, None, 0) as cq: qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex, e.IBV_QPT_RC) with QP(ctx, qia) as qp: assert qp.qp_state == e.IBV_QPS_RESET, 'RC QP should have been in RESET' qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex, e.IBV_QPT_UC) with QP(ctx, qia) as qp: assert qp.qp_state == e.IBV_QPS_RESET, 'UC QP should have been in RESET'
def create_qp(self): qp_caps = QPCap(max_recv_wr=self.num_msgs) qp_init_attr = QPInitAttr(qp_type=e.IBV_QPT_UD, cap=qp_caps, scq=self.cq, rcq=self.cq) qp_attr = QPAttr(port_num=self.ib_port) qp_attr.qkey = self.UD_QKEY qp_attr.pkey_index = self.UD_PKEY_INDEX self.qp = QP(self.pd, qp_init_attr, qp_attr)
def test_create_qp_no_attr(self): """ Test QP creation via ibv_create_qp without a QPAttr object proivded. Checked QP types are Raw Packet and UD. Raw Packet is skipped for non-root users / Infiniband link layer. """ for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: with CQ(ctx, 100, None, None, 0) as cq: for i in range(1, attr.phys_port_cnt + 1): qia = get_qp_init_attr(cq, attr) qia.qp_type = e.IBV_QPT_UD with QP(pd, qia) as qp: assert qp.qp_state == e.IBV_QPS_RESET, 'UD QP should have been in RESET' if is_eth(ctx, i) and is_root(): qia.qp_type = e.IBV_QPT_RAW_PACKET with QP(pd, qia) as qp: assert qp.qp_state == e.IBV_QPS_RESET, 'Raw Packet QP should have been in RESET'
def _create_rdma_objects(self): cq = CQ(self.pd_res.ctx, 100, None, None, 0) dev_attr = self.pd_res.ctx.query_device() qp_cap = u.random_qp_cap(dev_attr) qia = QPInitAttr(scq=cq, rcq=cq, cap=qp_cap) qia.qp_type = e.IBV_QPT_RC QP(self.pd_res.parent_domain, qia) srq_init_attr = SrqInitAttr(SrqAttr()) SRQ(self.pd_res.parent_domain, srq_init_attr)
def create_qps(self): qp_init_attr = self.create_qp_init_attr() qp_attr = self.create_qp_attr() qp_attr.qkey = self.UD_QKEY qp_attr.pkey_index = self.UD_PKEY_INDEX for _ in range(self.qp_count): qp = QP(self.pd, qp_init_attr, qp_attr) self.qps.append(qp) self.qps_num.append(qp.qp_num) self.psns.append(random.getrandbits(24))
def create_qp(self): """ Initializes self.qp with an RC QP. :return: None """ qp_caps = QPCap(max_recv_wr=self.num_msgs) qp_init_attr = QPInitAttr(qp_type=e.IBV_QPT_RC, scq=self.cq, rcq=self.cq, cap=qp_caps) qp_attr = QPAttr(port_num=self.ib_port) self.qp = QP(self.pd, qp_init_attr, qp_attr)
def test_create_qp_with_attr_connected(self): """ Test QP creation via ibv_create_qp without a QPAttr object proivded. QP type can be either RC or UC. """ for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: with CQ(ctx, 100, None, None, 0) as cq: qia = get_qp_init_attr(cq, [e.IBV_QPT_RC, e.IBV_QPT_UC], attr) with QP(pd, qia, QPAttr()) as qp: assert qp.qp_state == e.IBV_QPS_INIT
def test_create_qp_ex_with_attr(self): """ Test QP creation via ibv_create_qp_ex with a QPAttr object proivded. Checked QP types are Raw Packet and UD. Raw Packet is skipped for non-root users / Infiniband link layer. """ for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: with CQ(ctx, 100, None, None, 0) as cq: for i in range(1, attr.phys_port_cnt + 1): qpts = [e.IBV_QPT_UD, e.IBV_QPT_RAW_PACKET] \ if is_eth(ctx, i) else [e.IBV_QPT_UD] qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex, e.IBV_QPT_UD) with QP(ctx, qia, QPAttr()) as qp: assert qp.qp_state == e.IBV_QPS_RTS, 'UD QP should have been in RTS' if is_eth(ctx, i) and is_root(): qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex, e.IBV_QPT_RAW_PACKET) with QP(ctx, qia, QPAttr()) as qp: assert qp.qp_state == e.IBV_QPS_RTS, 'Raw Packet QP should have been in RTS'
def test_query_qp(self): """ Queries a QP after creation. Verifies that its properties are as expected. """ for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: with CQ(ctx, 100, None, None, 0) as cq: for i in range(1, attr.phys_port_cnt + 1): qpts = get_qp_types(ctx, i) is_ex = random.choice([True, False]) if is_ex: qia = get_qp_init_attr_ex(cq, pd, qpts, attr, attr_ex) else: qia = get_qp_init_attr(cq, qpts, attr) caps = qia.cap # Save them to verify values later qp = QP(ctx, qia) if is_ex else QP(pd, qia) attr, init_attr = qp.query(e.IBV_QP_CUR_STATE | e.IBV_QP_CAP) verify_qp_attrs(caps, e.IBV_QPS_RESET, init_attr, attr)
def test_query_qp(self): """ Queries a QP after creation. Verifies that its properties are as expected. """ for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: with CQ(ctx, 100, None, None, 0) as cq: for i in range(1, attr.phys_port_cnt + 1): qpts = get_qp_types(ctx, i) for qpt in qpts: # Extended QP qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex, qpt) caps = qia.cap # Save them to verify values later qp = QP(ctx, qia) qp_attr, qp_init_attr = qp.query(e.IBV_QP_CUR_STATE | e.IBV_QP_CAP) verify_qp_attrs(caps, e.IBV_QPS_RESET, qp_init_attr, qp_attr) # Legacy QP qia = get_qp_init_attr(cq, attr) qia.qp_type = qpt caps = qia.cap # Save them to verify values later qp = QP(pd, qia) qp_attr, qp_init_attr = qp.query(e.IBV_QP_CUR_STATE | e.IBV_QP_CAP) verify_qp_attrs(caps, e.IBV_QPS_RESET, qp_init_attr, qp_attr)
def test_query_qp(self): """ Queries a QP after creation. Verifies that its properties are as expected. """ for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: with CQ(ctx, 100, None, None, 0) as cq: for i in range(1, attr.phys_port_cnt + 1): qpts = get_qp_types(ctx, i) for qpt in qpts: # Extended QP qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex, qpt) caps = qia.cap # Save them to verify values later try: qp = QP(ctx, qia) except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest('Create QP with extended attrs is not supported') raise ex qp_attr, qp_init_attr = qp.query(e.IBV_QP_CUR_STATE | e.IBV_QP_CAP) verify_qp_attrs(caps, e.IBV_QPS_RESET, qp_init_attr, qp_attr) # Legacy QP qia = get_qp_init_attr(cq, attr) qia.qp_type = qpt caps = qia.cap # Save them to verify values later qp = QP(pd, qia) qp_attr, qp_init_attr = qp.query(e.IBV_QP_CUR_STATE | e.IBV_QP_CAP) verify_qp_attrs(caps, e.IBV_QPS_RESET, qp_init_attr, qp_attr)
def _create_rdma_objects(self): cq = CQ(self.pd_res.ctx, 100, None, None, 0) dev_attr = self.pd_res.ctx.query_device() qp_cap = u.random_qp_cap(dev_attr) qia = QPInitAttr(scq=cq, rcq=cq, cap=qp_cap) qia.qp_type = e.IBV_QPT_RC QP(self.pd_res.parent_domain, qia) srq_init_attr = SrqInitAttr(SrqAttr()) SRQ(self.pd_res.parent_domain, srq_init_attr) cq_init_attrs_ex = CqInitAttrEx( comp_mask=e.IBV_CQ_INIT_ATTR_MASK_PD, parent_domain=self.pd_res.parent_domain) CQEX(self.pd_res.ctx, cq_init_attrs_ex)
def create_qp(self, creator, qp_init_attr, is_ex, with_attr, port_num): """ Auxiliary function to create QP object. """ try: qp_attr = (None, QPAttr(port_num=port_num))[with_attr] return QP(creator, qp_init_attr, qp_attr) except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: with_str = ('without', 'with')[with_attr] + ('', ' extended')[is_ex] qp_type_str = pu.qp_type_to_str(qp_init_attr.qp_type) raise unittest.SkipTest(f'Create {qp_type_str} QP {with_str} attrs is not supported') raise ex
def test_create_qp_ex_no_attr(self): """ Test QP creation via ibv_create_qp_ex without a QPAttr object proivded. QP type can be either Raw Packet or UD. """ for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: with CQ(ctx, 100, None, None, 0) as cq: for i in range(1, attr.phys_port_cnt + 1): qpts = [e.IBV_QPT_UD, e.IBV_QPT_RAW_PACKET] \ if is_eth(ctx, i) else [e.IBV_QPT_UD] qia = get_qp_init_attr_ex(cq, pd, qpts, attr, attr_ex) with QP(ctx, qia) as qp: assert qp.qp_state == e.IBV_QPS_RESET
def create_qp(self, conn_idx=0): """ Create an rdmacm QP. If self.with_ext_qp is set, then an external CQ and QP will be created. In case that CQ is already created, it is used for the newly created QP. :param conn_idx: The connection index. """ cmid = self.child_id if self.passive else self.cmid if not self.with_ext_qp: cmid.create_qp(self.create_qp_init_attr()) else: self.create_cq(cmid) init_attr = self.create_qp_init_attr(rcq=self.cq, scq=self.cq) self.qps[conn_idx] = QP(cmid.pd, init_attr, QPAttr())
def create_qp(self): qp_caps = QPCap(max_recv_wr=self.num_msgs) qp_init_attr = QPInitAttr(qp_type=e.IBV_QPT_RC, scq=self.cq, rcq=self.cq, cap=qp_caps) qp_attr = QPAttr(port_num=self.ib_port) qp_access = e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_REMOTE_WRITE qp_attr.qp_access_flags = qp_access try: self.qp = QP(self.pd, qp_init_attr, qp_attr) except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest('Create RC QP is not supported') raise ex
def create_qp(self): qp_attr = QPAttr(port_num=self.ib_port) qp_attr.pkey_index = 0 for _ in range(self.qp_count): attr_ex = QPInitAttrEx(qp_type=e.IBV_QPT_XRC_RECV, comp_mask=e.IBV_QP_INIT_ATTR_XRCD, xrcd=self.xrcd) qp_attr.qp_access_flags = e.IBV_ACCESS_REMOTE_WRITE | \ e.IBV_ACCESS_REMOTE_READ recv_qp = QP(self.ctx, attr_ex, qp_attr) self.rqp_lst.append(recv_qp) send_qp = create_qp_ex(self, e.IBV_QPT_XRC_SEND, e.IBV_QP_EX_WITH_SEND) self.sqp_lst.append(send_qp) self.qps_num.append((recv_qp.qp_num, send_qp.qp_num)) self.psns.append(random.getrandbits(24))
def create_qps(self): """ Initializes self.qps with RC QPs. :return: None """ qp_init_attr = self.create_qp_init_attr() qp_attr = self.create_qp_attr() for _ in range(self.qp_count): try: qp = QP(self.pd, qp_init_attr, qp_attr) self.qps.append(qp) self.qps_num.append(qp.qp_num) self.psns.append(random.getrandbits(24)) except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest(f'Create QP type {qp_init_attr.qp_type} is not supported') raise ex
class CMResources(abc.ABC): """ CMResources class is an abstract base class which contains basic resources for RDMA CM communication. """ def __init__(self, addr=None, passive=None, **kwargs): """ :param addr: Local address to bind to. :param passive: Indicate if this CM is the passive CM. :param kwargs: Arguments: * *port* (str) Port number of the address * *with_ext_qp* (bool) If set, an external RC QP will be created and used by RDMACM * *port_space* (str) If set, indicates the CMIDs port space """ self.qp_init_attr = None self.passive = passive self.with_ext_qp = kwargs.get('with_ext_qp', False) self.port = kwargs.get('port') if kwargs.get('port') else '7471' self.port_space = kwargs.get('port_space', ce.RDMA_PS_TCP) self.remote_operation = kwargs.get('remote_op') self.qp_type = qp_type_per_ps[self.port_space] self.qp_init_attr = QPInitAttr(qp_type=self.qp_type, cap=QPCap()) self.connected = False # When passive side (server) listens to incoming connection requests, # for each new request it creates a new cmid which is used to establish # the connection with the remote side self.child_id = None self.msg_size = 1024 self.num_msgs = 10 self.channel = None self.cq = None self.qp = None self.mr = None self.remote_qpn = None self.ud_params = None if self.passive: self.ai = AddrInfo(src=addr, src_service=self.port, port_space=self.port_space, flags=ce.RAI_PASSIVE) else: self.ai = AddrInfo(src=addr, dst=addr, dst_service=self.port, port_space=self.port_space) def create_mr(self): cmid = self.child_id if self.passive else self.cmid mr_remote_function = { None: cmid.reg_msgs, 'read': cmid.reg_read, 'write': cmid.reg_write } self.mr = mr_remote_function[self.remote_operation](self.msg_size + GRH_SIZE) def create_event_channel(self): self.channel = CMEventChannel() def create_qp_init_attr(self, rcq=None, scq=None): return QPInitAttr(qp_type=self.qp_type, rcq=rcq, scq=scq, cap=QPCap(max_recv_wr=1)) def create_conn_param(self, qp_num=0): if self.with_ext_qp: qp_num = self.qp.qp_num return ConnParam(qp_num=qp_num) def set_ud_params(self, cm_event): if self.port_space == ce.RDMA_PS_UDP: self.ud_params = UDParam(cm_event) def my_qp_number(self): if self.with_ext_qp: return self.qp.qp_num else: cm = self.child_id if self.passive else self.cmid return cm.qpn def create_qp(self): """ Create an rdmacm QP. If self.with_ext_qp is set, then an external CQ and RC QP will be created and set in self.cq and self.qp respectively. """ cmid = self.child_id if self.passive else self.cmid if not self.with_ext_qp: cmid.create_qp(self.create_qp_init_attr()) else: self.cq = CQ(cmid.context, self.num_msgs, None, None, 0) init_attr = self.create_qp_init_attr(rcq=self.cq, scq=self.cq) self.qp = QP(cmid.pd, init_attr, QPAttr()) def modify_ext_qp_to_rts(self): cmid = self.child_id if self.passive else self.cmid attr, mask = cmid.init_qp_attr(e.IBV_QPS_INIT) self.qp.modify(attr, mask) attr, mask = cmid.init_qp_attr(e.IBV_QPS_RTR) self.qp.modify(attr, mask) attr, mask = cmid.init_qp_attr(e.IBV_QPS_RTS) self.qp.modify(attr, mask) @abc.abstractmethod def create_child_id(self, cm_event=None): pass
def create_qp(self): qia = QPInitAttr(e.IBV_QPT_RAW_PACKET, rcq=self.cq, scq=self.cq, cap=QPCap()) qp = QP(self.pd, qia) qp.to_init(QPAttr()) return qp
class CMResources: """ CMResources class is a base aggregator object which contains basic resources for RDMA CM communication. """ def __init__(self, **kwargs): """ :param kwargs: Arguments: * *src* (str) Local address to bind to (for passive side) * *dst* (str) Destination address to connect (for active side) * *port* (str) Port number of the address * *is_async* (bool) A flag which indicates if its asynchronous RDMACM * *with_ext_qp* (bool) If set, an external RC QP will be created and used by RDMACM """ src = kwargs.get('src') dst = kwargs.get('dst') self.is_server = True if dst is None else False self.qp_init_attr = None self.is_async = kwargs.get('is_async', False) self.with_ext_qp = kwargs.get('with_ext_qp', False) self.connected = False # When passive side (server) listens to incoming connection requests, # for each new request it creates a new cmid which is used to establish # the connection with the remote side self.child_id = None self.msg_size = 1024 self.num_msgs = 100 self.channel = None self.cq = None self.qp = None self.port = kwargs.get('port') if kwargs.get('port') else '7471' self.mr = None if self.is_server: self.ai = AddrInfo(src, None, self.port, ce.RDMA_PS_TCP, ce.RAI_PASSIVE) else: self.ai = AddrInfo(src, dst, self.port, ce.RDMA_PS_TCP) if self.is_async: self.create_event_channel() self.cmid = CMID(creator=self.channel) else: self.cmid = CMID(creator=self.ai, qp_init_attr=self.create_qp_init_attr()) def create_mr(self): if self.is_server: self.mr = self.child_id.reg_msgs(self.msg_size) else: self.mr = self.cmid.reg_msgs(self.msg_size) def create_event_channel(self): self.channel = CMEventChannel() @staticmethod def create_qp_init_attr(rcq=None, scq=None): return QPInitAttr(qp_type=e.IBV_QPT_RC, rcq=rcq, scq=scq, cap=QPCap(max_recv_wr=1)) @staticmethod def create_conn_param(qp_num=0): return ConnParam(qp_num=qp_num) def create_child_id(self, cm_event=None): if not self.is_server: raise PyverbsUserError( 'create_child_id can be used only in passive side') if self.is_async: self.child_id = CMID(creator=cm_event, listen_id=self.cmid) else: self.child_id = self.cmid.get_request() def create_qp(self): """ Create a rdmacm QP. If self.with_ext_qp is set, then an external CQ and RC QP will be created and set in self.cq and self.qp respectively. """ cmid = self.child_id if self.is_server else self.cmid if not self.with_ext_qp: cmid.create_qp(self.create_qp_init_attr()) else: self.cq = CQ(cmid.context, self.num_msgs, None, None, 0) init_attr = self.create_qp_init_attr(rcq=self.cq, scq=self.cq) self.qp = QP(cmid.pd, init_attr, QPAttr()) def modify_ext_qp_to_rts(self): cmid = self.child_id if self.is_server else self.cmid attr, mask = cmid.init_qp_attr(e.IBV_QPS_INIT) self.qp.modify(attr, mask) attr, mask = cmid.init_qp_attr(e.IBV_QPS_RTR) self.qp.modify(attr, mask) attr, mask = cmid.init_qp_attr(e.IBV_QPS_RTS) self.qp.modify(attr, mask)
def test_modify_qp(self): """ Queries a QP after calling modify(). Verifies that its properties are as expected. """ for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: with CQ(ctx, 100, None, None, 0) as cq: # Extended QP qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex, e.IBV_QPT_UD) try: qp = QP(ctx, qia) except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest('Create QP with extended attrs is not supported') raise ex qa = QPAttr() qa.qkey = 0x123 qp.to_init(qa) qp_attr, qp_iattr = qp.query(e.IBV_QP_QKEY) assert qp_attr.qkey == qa.qkey, 'Extended QP, QKey is not as expected' qp.to_rtr(qa) qa.sq_psn = 0x45 qp.to_rts(qa) qp_attr, qp_iattr = qp.query(e.IBV_QP_SQ_PSN) assert qp_attr.sq_psn == qa.sq_psn, 'Extended QP, SQ PSN is not as expected' qa.qp_state = e.IBV_QPS_RESET qp.modify(qa, e.IBV_QP_STATE) assert qp.qp_state == e.IBV_QPS_RESET, 'Extended QP, QP state is not as expected' # Legacy QP qia = get_qp_init_attr(cq, attr) qp = QP(pd, qia) qa = QPAttr() qa.qkey = 0x123 qp.to_init(qa) qp_attr, qp_iattr = qp.query(e.IBV_QP_QKEY) assert qp_attr.qkey == qa.qkey, 'Legacy QP, QKey is not as expected' qp.to_rtr(qa) qa.sq_psn = 0x45 qp.to_rts(qa) qp_attr, qp_iattr = qp.query(e.IBV_QP_SQ_PSN) assert qp_attr.sq_psn == qa.sq_psn, 'Legacy QP, SQ PSN is not as expected' qa.qp_state = e.IBV_QPS_RESET qp.modify(qa, e.IBV_QP_STATE) assert qp.qp_state == e.IBV_QPS_RESET, 'Legacy QP, QP state is not as expected'