def test_create_qp_no_attr(self):
"""
Test QP creation via ibv_create_qp without a QPAttr object proivded.
Checked QP types are Raw Packet and UD. Raw Packet is skipped for
non-root users / Infiniband link layer.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
for i in range(1, attr.phys_port_cnt + 1):
qia = get_qp_init_attr(cq, attr)
qia.qp_type = e.IBV_QPT_UD
with QP(pd, qia) as qp:
assert qp.qp_state == e.IBV_QPS_RESET, 'UD QP should have been in RESET'
if u.is_eth(ctx, i) and u.is_root():
qia.qp_type = e.IBV_QPT_RAW_PACKET
try:
with QP(pd, qia) as qp:
assert qp.qp_state == e.IBV_QPS_RESET, 'Raw Packet QP should have been in RESET'
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest(
"Create Raw Packet QP is not supported"
)
raise ex
def test_create_qp_no_attr_connected(self):
"""
Test QP creation via ibv_create_qp without a QPAttr object proivded.
Checked QP types are RC and UC.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
qia = get_qp_init_attr(cq, attr)
qia.qp_type = e.IBV_QPT_RC
try:
with QP(pd, qia) as qp:
assert qp.qp_state == e.IBV_QPS_RESET, 'RC QP should have been in RESET'
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest(
'Create QP with RC attrs is not supported')
raise ex
qia.qp_type = e.IBV_QPT_UC
try:
with QP(pd, qia) as qp:
assert qp.qp_state == e.IBV_QPS_RESET, 'UC QP should have been in RESET'
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest(
'Create QP with UC attrs is not supported')
raise ex
def create_qps(self):
"""
Initializes self.qps with an XRC SEND/RECV QPs.
:return: None
"""
qp_attr = QPAttr(port_num=self.ib_port)
qp_attr.pkey_index = 0
for _ in range(self.qp_count):
attr_ex = QPInitAttrEx(qp_type=e.IBV_QPT_XRC_RECV,
comp_mask=e.IBV_QP_INIT_ATTR_XRCD,
xrcd=self.xrcd)
qp_attr.qp_access_flags = e.IBV_ACCESS_REMOTE_WRITE | \
e.IBV_ACCESS_REMOTE_READ
recv_qp = QP(self.ctx, attr_ex, qp_attr)
self.rqp_lst.append(recv_qp)
qp_caps = QPCap(max_send_wr=self.num_msgs, max_recv_sge=0,
max_recv_wr=0)
attr_ex = QPInitAttrEx(qp_type=e.IBV_QPT_XRC_SEND, sq_sig_all=1,
comp_mask=e.IBV_QP_INIT_ATTR_PD,
pd=self.pd, scq=self.cq, cap=qp_caps)
qp_attr.qp_access_flags = 0
send_qp =QP(self.ctx, attr_ex, qp_attr)
self.sqp_lst.append(send_qp)
self.qps_num.append((recv_qp.qp_num, send_qp.qp_num))
self.psns.append(random.getrandbits(24))
def test_create_qp_ex_with_attr(self):
"""
Test QP creation via ibv_create_qp_ex with a QPAttr object proivded.
Checked QP types are Raw Packet and UD. Raw Packet is skipped for
non-root users / Infiniband link layer.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
for i in range(1, attr.phys_port_cnt + 1):
qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex,
e.IBV_QPT_UD)
try:
with QP(ctx, qia, QPAttr()) as qp:
assert qp.qp_state == e.IBV_QPS_RTS, 'UD QP should have been in RTS'
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create QP with extended attrs is not supported')
raise ex
if is_eth(ctx, i) and is_root():
qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex,
e.IBV_QPT_RAW_PACKET)
try:
with QP(ctx, qia, QPAttr()) as qp:
assert qp.qp_state == e.IBV_QPS_RTS, 'Raw Packet QP should have been in RTS'
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create QP with extended attrs is not supported')
raise ex
def test_create_qp_ex_with_attr_connected(self):
"""
Test QP creation via ibv_create_qp_ex with a QPAttr object proivded.
Checked QP type are RC and UC.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex,
e.IBV_QPT_RC)
try:
with QP(ctx, qia, QPAttr()) as qp:
assert qp.qp_state == e.IBV_QPS_INIT, 'RC QP should have been in INIT'
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create QP with extended attrs is not supported')
raise ex
qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex,
e.IBV_QPT_UC)
try:
with QP(ctx, qia, QPAttr()) as qp:
assert qp.qp_state == e.IBV_QPS_INIT, 'UC QP should have been in INIT'
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create QP with extended attrs is not supported')
raise ex
def create_qp(self):
"""
Create an rdmacm QP. If self.with_ext_qp is set, then an external CQ and
RC QP will be created and set in self.cq and self.qp
respectively.
"""
cmid = self.child_id if self.passive else self.cmid
if not self.with_ext_qp:
cmid.create_qp(self.create_qp_init_attr())
else:
self.cq = CQ(cmid.context, self.num_msgs, None, None, 0)
init_attr = self.create_qp_init_attr(rcq=self.cq, scq=self.cq)
self.qp = QP(cmid.pd, init_attr, QPAttr())
def create_qp(self):
qia = QPInitAttr(e.IBV_QPT_RAW_PACKET,
rcq=self.cq,
scq=self.cq,
cap=QPCap())
try:
qp = QP(self.pd, qia)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest(
"Create Raw Packet QP is not supported")
raise ex
qp.to_init(QPAttr())
return qp
def test_create_qp_with_attr_connected(self):
"""
Test QP creation via ibv_create_qp without a QPAttr object proivded.
Checked QP types are RC and UC.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
qia = get_qp_init_attr(cq, attr)
qia.qp_type = e.IBV_QPT_RC
with QP(pd, qia, QPAttr()) as qp:
assert qp.qp_state == e.IBV_QPS_INIT, 'RC QP should have been in INIT'
qia.qp_type = e.IBV_QPT_UC
with QP(pd, qia, QPAttr()) as qp:
assert qp.qp_state == e.IBV_QPS_INIT, 'UC QP should have been in INIT'
def test_create_qp_ex_no_attr_connected(self):
"""
Test QP creation via ibv_create_qp_ex without a QPAttr object proivded.
Checked QP types are RC and UC.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex,
e.IBV_QPT_RC)
with QP(ctx, qia) as qp:
assert qp.qp_state == e.IBV_QPS_RESET, 'RC QP should have been in RESET'
qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex,
e.IBV_QPT_UC)
with QP(ctx, qia) as qp:
assert qp.qp_state == e.IBV_QPS_RESET, 'UC QP should have been in RESET'
def create_qp(self):
qp_caps = QPCap(max_recv_wr=self.num_msgs)
qp_init_attr = QPInitAttr(qp_type=e.IBV_QPT_UD, cap=qp_caps,
scq=self.cq, rcq=self.cq)
qp_attr = QPAttr(port_num=self.ib_port)
qp_attr.qkey = self.UD_QKEY
qp_attr.pkey_index = self.UD_PKEY_INDEX
self.qp = QP(self.pd, qp_init_attr, qp_attr)
def test_create_qp_no_attr(self):
"""
Test QP creation via ibv_create_qp without a QPAttr object proivded.
Checked QP types are Raw Packet and UD. Raw Packet is skipped for
non-root users / Infiniband link layer.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
for i in range(1, attr.phys_port_cnt + 1):
qia = get_qp_init_attr(cq, attr)
qia.qp_type = e.IBV_QPT_UD
with QP(pd, qia) as qp:
assert qp.qp_state == e.IBV_QPS_RESET, 'UD QP should have been in RESET'
if is_eth(ctx, i) and is_root():
qia.qp_type = e.IBV_QPT_RAW_PACKET
with QP(pd, qia) as qp:
assert qp.qp_state == e.IBV_QPS_RESET, 'Raw Packet QP should have been in RESET'
def _create_rdma_objects(self):
cq = CQ(self.pd_res.ctx, 100, None, None, 0)
dev_attr = self.pd_res.ctx.query_device()
qp_cap = u.random_qp_cap(dev_attr)
qia = QPInitAttr(scq=cq, rcq=cq, cap=qp_cap)
qia.qp_type = e.IBV_QPT_RC
QP(self.pd_res.parent_domain, qia)
srq_init_attr = SrqInitAttr(SrqAttr())
SRQ(self.pd_res.parent_domain, srq_init_attr)
def create_qps(self):
qp_init_attr = self.create_qp_init_attr()
qp_attr = self.create_qp_attr()
qp_attr.qkey = self.UD_QKEY
qp_attr.pkey_index = self.UD_PKEY_INDEX
for _ in range(self.qp_count):
qp = QP(self.pd, qp_init_attr, qp_attr)
self.qps.append(qp)
self.qps_num.append(qp.qp_num)
self.psns.append(random.getrandbits(24))
def create_qp(self):
"""
Initializes self.qp with an RC QP.
:return: None
"""
qp_caps = QPCap(max_recv_wr=self.num_msgs)
qp_init_attr = QPInitAttr(qp_type=e.IBV_QPT_RC, scq=self.cq,
rcq=self.cq, cap=qp_caps)
qp_attr = QPAttr(port_num=self.ib_port)
self.qp = QP(self.pd, qp_init_attr, qp_attr)
def test_create_qp_with_attr_connected(self):
"""
Test QP creation via ibv_create_qp without a QPAttr object proivded.
QP type can be either RC or UC.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
qia = get_qp_init_attr(cq, [e.IBV_QPT_RC, e.IBV_QPT_UC],
attr)
with QP(pd, qia, QPAttr()) as qp:
assert qp.qp_state == e.IBV_QPS_INIT
def test_create_qp_ex_with_attr(self):
"""
Test QP creation via ibv_create_qp_ex with a QPAttr object proivded.
Checked QP types are Raw Packet and UD. Raw Packet is skipped for
non-root users / Infiniband link layer.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
for i in range(1, attr.phys_port_cnt + 1):
qpts = [e.IBV_QPT_UD, e.IBV_QPT_RAW_PACKET] \
if is_eth(ctx, i) else [e.IBV_QPT_UD]
qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex,
e.IBV_QPT_UD)
with QP(ctx, qia, QPAttr()) as qp:
assert qp.qp_state == e.IBV_QPS_RTS, 'UD QP should have been in RTS'
if is_eth(ctx, i) and is_root():
qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex,
e.IBV_QPT_RAW_PACKET)
with QP(ctx, qia, QPAttr()) as qp:
assert qp.qp_state == e.IBV_QPS_RTS, 'Raw Packet QP should have been in RTS'
def test_query_qp(self):
"""
Queries a QP after creation. Verifies that its properties are as
expected.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
for i in range(1, attr.phys_port_cnt + 1):
qpts = get_qp_types(ctx, i)
is_ex = random.choice([True, False])
if is_ex:
qia = get_qp_init_attr_ex(cq, pd, qpts, attr,
attr_ex)
else:
qia = get_qp_init_attr(cq, qpts, attr)
caps = qia.cap # Save them to verify values later
qp = QP(ctx, qia) if is_ex else QP(pd, qia)
attr, init_attr = qp.query(e.IBV_QP_CUR_STATE
| e.IBV_QP_CAP)
verify_qp_attrs(caps, e.IBV_QPS_RESET, init_attr, attr)
def test_query_qp(self):
"""
Queries a QP after creation. Verifies that its properties are as
expected.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
for i in range(1, attr.phys_port_cnt + 1):
qpts = get_qp_types(ctx, i)
for qpt in qpts:
# Extended QP
qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex,
qpt)
caps = qia.cap # Save them to verify values later
qp = QP(ctx, qia)
qp_attr, qp_init_attr = qp.query(e.IBV_QP_CUR_STATE
| e.IBV_QP_CAP)
verify_qp_attrs(caps, e.IBV_QPS_RESET,
qp_init_attr, qp_attr)
# Legacy QP
qia = get_qp_init_attr(cq, attr)
qia.qp_type = qpt
caps = qia.cap # Save them to verify values later
qp = QP(pd, qia)
qp_attr, qp_init_attr = qp.query(e.IBV_QP_CUR_STATE
| e.IBV_QP_CAP)
verify_qp_attrs(caps, e.IBV_QPS_RESET,
qp_init_attr, qp_attr)
def test_query_qp(self):
"""
Queries a QP after creation. Verifies that its properties are as
expected.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
for i in range(1, attr.phys_port_cnt + 1):
qpts = get_qp_types(ctx, i)
for qpt in qpts:
# Extended QP
qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex,
qpt)
caps = qia.cap # Save them to verify values later
try:
qp = QP(ctx, qia)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create QP with extended attrs is not supported')
raise ex
qp_attr, qp_init_attr = qp.query(e.IBV_QP_CUR_STATE |
e.IBV_QP_CAP)
verify_qp_attrs(caps, e.IBV_QPS_RESET, qp_init_attr,
qp_attr)
# Legacy QP
qia = get_qp_init_attr(cq, attr)
qia.qp_type = qpt
caps = qia.cap # Save them to verify values later
qp = QP(pd, qia)
qp_attr, qp_init_attr = qp.query(e.IBV_QP_CUR_STATE |
e.IBV_QP_CAP)
verify_qp_attrs(caps, e.IBV_QPS_RESET, qp_init_attr,
qp_attr)
def _create_rdma_objects(self):
cq = CQ(self.pd_res.ctx, 100, None, None, 0)
dev_attr = self.pd_res.ctx.query_device()
qp_cap = u.random_qp_cap(dev_attr)
qia = QPInitAttr(scq=cq, rcq=cq, cap=qp_cap)
qia.qp_type = e.IBV_QPT_RC
QP(self.pd_res.parent_domain, qia)
srq_init_attr = SrqInitAttr(SrqAttr())
SRQ(self.pd_res.parent_domain, srq_init_attr)
cq_init_attrs_ex = CqInitAttrEx(
comp_mask=e.IBV_CQ_INIT_ATTR_MASK_PD,
parent_domain=self.pd_res.parent_domain)
CQEX(self.pd_res.ctx, cq_init_attrs_ex)
def create_qp(self, creator, qp_init_attr, is_ex, with_attr, port_num):
"""
Auxiliary function to create QP object.
"""
try:
qp_attr = (None, QPAttr(port_num=port_num))[with_attr]
return QP(creator, qp_init_attr, qp_attr)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
with_str = ('without', 'with')[with_attr] + ('', ' extended')[is_ex]
qp_type_str = pu.qp_type_to_str(qp_init_attr.qp_type)
raise unittest.SkipTest(f'Create {qp_type_str} QP {with_str} attrs is not supported')
raise ex
def test_create_qp_ex_no_attr(self):
"""
Test QP creation via ibv_create_qp_ex without a QPAttr object proivded.
QP type can be either Raw Packet or UD.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
for i in range(1, attr.phys_port_cnt + 1):
qpts = [e.IBV_QPT_UD, e.IBV_QPT_RAW_PACKET] \
if is_eth(ctx, i) else [e.IBV_QPT_UD]
qia = get_qp_init_attr_ex(cq, pd, qpts, attr, attr_ex)
with QP(ctx, qia) as qp:
assert qp.qp_state == e.IBV_QPS_RESET
def create_qp(self, conn_idx=0):
"""
Create an rdmacm QP. If self.with_ext_qp is set, then an external CQ and
QP will be created. In case that CQ is already created, it is used
for the newly created QP.
:param conn_idx: The connection index.
"""
cmid = self.child_id if self.passive else self.cmid
if not self.with_ext_qp:
cmid.create_qp(self.create_qp_init_attr())
else:
self.create_cq(cmid)
init_attr = self.create_qp_init_attr(rcq=self.cq, scq=self.cq)
self.qps[conn_idx] = QP(cmid.pd, init_attr, QPAttr())
def create_qp(self):
qp_caps = QPCap(max_recv_wr=self.num_msgs)
qp_init_attr = QPInitAttr(qp_type=e.IBV_QPT_RC,
scq=self.cq,
rcq=self.cq,
cap=qp_caps)
qp_attr = QPAttr(port_num=self.ib_port)
qp_access = e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_REMOTE_WRITE
qp_attr.qp_access_flags = qp_access
try:
self.qp = QP(self.pd, qp_init_attr, qp_attr)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create RC QP is not supported')
raise ex
def create_qp(self):
qp_attr = QPAttr(port_num=self.ib_port)
qp_attr.pkey_index = 0
for _ in range(self.qp_count):
attr_ex = QPInitAttrEx(qp_type=e.IBV_QPT_XRC_RECV,
comp_mask=e.IBV_QP_INIT_ATTR_XRCD,
xrcd=self.xrcd)
qp_attr.qp_access_flags = e.IBV_ACCESS_REMOTE_WRITE | \
e.IBV_ACCESS_REMOTE_READ
recv_qp = QP(self.ctx, attr_ex, qp_attr)
self.rqp_lst.append(recv_qp)
send_qp = create_qp_ex(self, e.IBV_QPT_XRC_SEND, e.IBV_QP_EX_WITH_SEND)
self.sqp_lst.append(send_qp)
self.qps_num.append((recv_qp.qp_num, send_qp.qp_num))
self.psns.append(random.getrandbits(24))
def create_qps(self):
"""
Initializes self.qps with RC QPs.
:return: None
"""
qp_init_attr = self.create_qp_init_attr()
qp_attr = self.create_qp_attr()
for _ in range(self.qp_count):
try:
qp = QP(self.pd, qp_init_attr, qp_attr)
self.qps.append(qp)
self.qps_num.append(qp.qp_num)
self.psns.append(random.getrandbits(24))
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest(f'Create QP type {qp_init_attr.qp_type} is not supported')
raise ex
class CMResources(abc.ABC):
"""
CMResources class is an abstract base class which contains basic resources
for RDMA CM communication.
"""
def __init__(self, addr=None, passive=None, **kwargs):
"""
:param addr: Local address to bind to.
:param passive: Indicate if this CM is the passive CM.
:param kwargs: Arguments:
* *port* (str)
Port number of the address
* *with_ext_qp* (bool)
If set, an external RC QP will be created and used by RDMACM
* *port_space* (str)
If set, indicates the CMIDs port space
"""
self.qp_init_attr = None
self.passive = passive
self.with_ext_qp = kwargs.get('with_ext_qp', False)
self.port = kwargs.get('port') if kwargs.get('port') else '7471'
self.port_space = kwargs.get('port_space', ce.RDMA_PS_TCP)
self.remote_operation = kwargs.get('remote_op')
self.qp_type = qp_type_per_ps[self.port_space]
self.qp_init_attr = QPInitAttr(qp_type=self.qp_type, cap=QPCap())
self.connected = False
# When passive side (server) listens to incoming connection requests,
# for each new request it creates a new cmid which is used to establish
# the connection with the remote side
self.child_id = None
self.msg_size = 1024
self.num_msgs = 10
self.channel = None
self.cq = None
self.qp = None
self.mr = None
self.remote_qpn = None
self.ud_params = None
if self.passive:
self.ai = AddrInfo(src=addr,
src_service=self.port,
port_space=self.port_space,
flags=ce.RAI_PASSIVE)
else:
self.ai = AddrInfo(src=addr,
dst=addr,
dst_service=self.port,
port_space=self.port_space)
def create_mr(self):
cmid = self.child_id if self.passive else self.cmid
mr_remote_function = {
None: cmid.reg_msgs,
'read': cmid.reg_read,
'write': cmid.reg_write
}
self.mr = mr_remote_function[self.remote_operation](self.msg_size +
GRH_SIZE)
def create_event_channel(self):
self.channel = CMEventChannel()
def create_qp_init_attr(self, rcq=None, scq=None):
return QPInitAttr(qp_type=self.qp_type,
rcq=rcq,
scq=scq,
cap=QPCap(max_recv_wr=1))
def create_conn_param(self, qp_num=0):
if self.with_ext_qp:
qp_num = self.qp.qp_num
return ConnParam(qp_num=qp_num)
def set_ud_params(self, cm_event):
if self.port_space == ce.RDMA_PS_UDP:
self.ud_params = UDParam(cm_event)
def my_qp_number(self):
if self.with_ext_qp:
return self.qp.qp_num
else:
cm = self.child_id if self.passive else self.cmid
return cm.qpn
def create_qp(self):
"""
Create an rdmacm QP. If self.with_ext_qp is set, then an external CQ and
RC QP will be created and set in self.cq and self.qp
respectively.
"""
cmid = self.child_id if self.passive else self.cmid
if not self.with_ext_qp:
cmid.create_qp(self.create_qp_init_attr())
else:
self.cq = CQ(cmid.context, self.num_msgs, None, None, 0)
init_attr = self.create_qp_init_attr(rcq=self.cq, scq=self.cq)
self.qp = QP(cmid.pd, init_attr, QPAttr())
def modify_ext_qp_to_rts(self):
cmid = self.child_id if self.passive else self.cmid
attr, mask = cmid.init_qp_attr(e.IBV_QPS_INIT)
self.qp.modify(attr, mask)
attr, mask = cmid.init_qp_attr(e.IBV_QPS_RTR)
self.qp.modify(attr, mask)
attr, mask = cmid.init_qp_attr(e.IBV_QPS_RTS)
self.qp.modify(attr, mask)
@abc.abstractmethod
def create_child_id(self, cm_event=None):
pass
def create_qp(self):
qia = QPInitAttr(e.IBV_QPT_RAW_PACKET, rcq=self.cq, scq=self.cq,
cap=QPCap())
qp = QP(self.pd, qia)
qp.to_init(QPAttr())
return qp
class CMResources:
"""
CMResources class is a base aggregator object which contains basic
resources for RDMA CM communication.
"""
def __init__(self, **kwargs):
"""
:param kwargs: Arguments:
* *src* (str)
Local address to bind to (for passive side)
* *dst* (str)
Destination address to connect (for active side)
* *port* (str)
Port number of the address
* *is_async* (bool)
A flag which indicates if its asynchronous RDMACM
* *with_ext_qp* (bool)
If set, an external RC QP will be created and used by RDMACM
"""
src = kwargs.get('src')
dst = kwargs.get('dst')
self.is_server = True if dst is None else False
self.qp_init_attr = None
self.is_async = kwargs.get('is_async', False)
self.with_ext_qp = kwargs.get('with_ext_qp', False)
self.connected = False
# When passive side (server) listens to incoming connection requests,
# for each new request it creates a new cmid which is used to establish
# the connection with the remote side
self.child_id = None
self.msg_size = 1024
self.num_msgs = 100
self.channel = None
self.cq = None
self.qp = None
self.port = kwargs.get('port') if kwargs.get('port') else '7471'
self.mr = None
if self.is_server:
self.ai = AddrInfo(src, None, self.port, ce.RDMA_PS_TCP,
ce.RAI_PASSIVE)
else:
self.ai = AddrInfo(src, dst, self.port, ce.RDMA_PS_TCP)
if self.is_async:
self.create_event_channel()
self.cmid = CMID(creator=self.channel)
else:
self.cmid = CMID(creator=self.ai,
qp_init_attr=self.create_qp_init_attr())
def create_mr(self):
if self.is_server:
self.mr = self.child_id.reg_msgs(self.msg_size)
else:
self.mr = self.cmid.reg_msgs(self.msg_size)
def create_event_channel(self):
self.channel = CMEventChannel()
@staticmethod
def create_qp_init_attr(rcq=None, scq=None):
return QPInitAttr(qp_type=e.IBV_QPT_RC,
rcq=rcq,
scq=scq,
cap=QPCap(max_recv_wr=1))
@staticmethod
def create_conn_param(qp_num=0):
return ConnParam(qp_num=qp_num)
def create_child_id(self, cm_event=None):
if not self.is_server:
raise PyverbsUserError(
'create_child_id can be used only in passive side')
if self.is_async:
self.child_id = CMID(creator=cm_event, listen_id=self.cmid)
else:
self.child_id = self.cmid.get_request()
def create_qp(self):
"""
Create a rdmacm QP. If self.with_ext_qp is set, then an external CQ and
RC QP will be created and set in self.cq and self.qp
respectively.
"""
cmid = self.child_id if self.is_server else self.cmid
if not self.with_ext_qp:
cmid.create_qp(self.create_qp_init_attr())
else:
self.cq = CQ(cmid.context, self.num_msgs, None, None, 0)
init_attr = self.create_qp_init_attr(rcq=self.cq, scq=self.cq)
self.qp = QP(cmid.pd, init_attr, QPAttr())
def modify_ext_qp_to_rts(self):
cmid = self.child_id if self.is_server else self.cmid
attr, mask = cmid.init_qp_attr(e.IBV_QPS_INIT)
self.qp.modify(attr, mask)
attr, mask = cmid.init_qp_attr(e.IBV_QPS_RTR)
self.qp.modify(attr, mask)
attr, mask = cmid.init_qp_attr(e.IBV_QPS_RTS)
self.qp.modify(attr, mask)
def test_modify_qp(self):
"""
Queries a QP after calling modify(). Verifies that its properties are
as expected.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
# Extended QP
qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex, e.IBV_QPT_UD)
try:
qp = QP(ctx, qia)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create QP with extended attrs is not supported')
raise ex
qa = QPAttr()
qa.qkey = 0x123
qp.to_init(qa)
qp_attr, qp_iattr = qp.query(e.IBV_QP_QKEY)
assert qp_attr.qkey == qa.qkey, 'Extended QP, QKey is not as expected'
qp.to_rtr(qa)
qa.sq_psn = 0x45
qp.to_rts(qa)
qp_attr, qp_iattr = qp.query(e.IBV_QP_SQ_PSN)
assert qp_attr.sq_psn == qa.sq_psn, 'Extended QP, SQ PSN is not as expected'
qa.qp_state = e.IBV_QPS_RESET
qp.modify(qa, e.IBV_QP_STATE)
assert qp.qp_state == e.IBV_QPS_RESET, 'Extended QP, QP state is not as expected'
# Legacy QP
qia = get_qp_init_attr(cq, attr)
qp = QP(pd, qia)
qa = QPAttr()
qa.qkey = 0x123
qp.to_init(qa)
qp_attr, qp_iattr = qp.query(e.IBV_QP_QKEY)
assert qp_attr.qkey == qa.qkey, 'Legacy QP, QKey is not as expected'
qp.to_rtr(qa)
qa.sq_psn = 0x45
qp.to_rts(qa)
qp_attr, qp_iattr = qp.query(e.IBV_QP_SQ_PSN)
assert qp_attr.sq_psn == qa.sq_psn, 'Legacy QP, SQ PSN is not as expected'
qa.qp_state = e.IBV_QPS_RESET
qp.modify(qa, e.IBV_QP_STATE)
assert qp.qp_state == e.IBV_QPS_RESET, 'Legacy QP, QP state is not as expected'
