def create_qp_ex(agr_obj, qp_type, send_flags):
if qp_type == e.IBV_QPT_XRC_SEND:
cap = QPCap(max_send_wr=agr_obj.num_msgs, max_recv_wr=0, max_recv_sge=0,
max_send_sge=1)
else:
cap = QPCap(max_send_wr=agr_obj.num_msgs, max_recv_wr=agr_obj.num_msgs,
max_recv_sge=1, max_send_sge=1)
qia = QPInitAttrEx(cap=cap, qp_type=qp_type, scq=agr_obj.cq,
rcq=agr_obj.cq, pd=agr_obj.pd, send_ops_flags=send_flags,
comp_mask=e.IBV_QP_INIT_ATTR_PD |
e.IBV_QP_INIT_ATTR_SEND_OPS_FLAGS)
qp_attr = QPAttr(port_num=agr_obj.ib_port)
if qp_type == e.IBV_QPT_UD:
qp_attr.qkey = agr_obj.UD_QKEY
qp_attr.pkey_index = agr_obj.UD_PKEY_INDEX
if qp_type == e.IBV_QPT_RC:
qp_attr.qp_access_flags = e.IBV_ACCESS_REMOTE_WRITE | \
e.IBV_ACCESS_REMOTE_READ | \
e.IBV_ACCESS_REMOTE_ATOMIC
try:
# We don't have capability bits for this
qp = QPEx(agr_obj.ctx, qia, qp_attr)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Extended QP is not supported on this device')
raise ex
return qp
def create_qps(self):
qp_cap = QPCap(max_recv_wr=self.num_msgs,
max_send_wr=self.num_msgs,
max_recv_sge=1,
max_send_sge=1)
comp_mask = e.IBV_QP_INIT_ATTR_PD
if self.send_ops_flags:
comp_mask |= e.IBV_QP_INIT_ATTR_SEND_OPS_FLAGS
qp_init_attr_ex = QPInitAttrEx(cap=qp_cap,
qp_type=e.IBV_QPT_DRIVER,
scq=self.cq,
rcq=self.cq,
pd=self.pd,
send_ops_flags=self.send_ops_flags,
comp_mask=comp_mask)
efa_init_attr_ex = efa.EfaQPInitAttr()
efa_init_attr_ex.driver_qp_type = efa_e.EFADV_QP_DRIVER_TYPE_SRD
try:
for _ in range(self.qp_count):
qp = efa.SRDQPEx(self.ctx, qp_init_attr_ex, efa_init_attr_ex)
self.qps.append(qp)
self.qps_num.append(qp.qp_num)
self.psns.append(random.getrandbits(24))
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest(
'Extended SRD QP is not supported on this device')
raise ex
def create_qps(self):
"""
Initializes self.qps with an XRC SEND/RECV QPs.
:return: None
"""
qp_attr = QPAttr(port_num=self.ib_port)
qp_attr.pkey_index = 0
for _ in range(self.qp_count):
attr_ex = QPInitAttrEx(qp_type=e.IBV_QPT_XRC_RECV,
comp_mask=e.IBV_QP_INIT_ATTR_XRCD,
xrcd=self.xrcd)
qp_attr.qp_access_flags = e.IBV_ACCESS_REMOTE_WRITE | \
e.IBV_ACCESS_REMOTE_READ
recv_qp = QP(self.ctx, attr_ex, qp_attr)
self.rqp_lst.append(recv_qp)
qp_caps = QPCap(max_send_wr=self.num_msgs, max_recv_sge=0,
max_recv_wr=0)
attr_ex = QPInitAttrEx(qp_type=e.IBV_QPT_XRC_SEND, sq_sig_all=1,
comp_mask=e.IBV_QP_INIT_ATTR_PD,
pd=self.pd, scq=self.cq, cap=qp_caps)
qp_attr.qp_access_flags = 0
send_qp =QP(self.ctx, attr_ex, qp_attr)
self.sqp_lst.append(send_qp)
self.qps_num.append((recv_qp.qp_num, send_qp.qp_num))
self.psns.append(random.getrandbits(24))
def __init__(self, addr=None, passive=None, **kwargs):
"""
:param addr: Local address to bind to.
:param passive: Indicate if this CM is the passive CM.
:param kwargs: Arguments:
* *port* (str)
Port number of the address
* *with_ext_qp* (bool)
If set, an external RC QP will be created and used by RDMACM
"""
self.qp_init_attr = None
self.passive = passive
self.with_ext_qp = kwargs.get('with_ext_qp', False)
self.port = kwargs.get('port') if kwargs.get('port') else '7471'
self.port_space = ce.RDMA_PS_TCP
self.qp_type = e.IBV_QPT_RC
self.qp_init_attr = QPInitAttr(qp_type=self.qp_type, cap=QPCap())
self.connected = False
# When passive side (server) listens to incoming connection requests,
# for each new request it creates a new cmid which is used to establish
# the connection with the remote side
self.child_id = None
self.msg_size = 1024
self.num_msgs = 10
self.channel = None
self.cq = None
self.qp = None
self.mr = None
if self.passive:
self.ai = AddrInfo(addr, None, self.port, self.port_space,
ce.RAI_PASSIVE)
else:
self.ai = AddrInfo(addr, addr, self.port, self.port_space)
def create_qp(self):
qp_caps = QPCap(max_recv_wr=self.num_msgs)
qp_init_attr = QPInitAttr(qp_type=e.IBV_QPT_UD, cap=qp_caps,
scq=self.cq, rcq=self.cq)
qp_attr = QPAttr(port_num=self.ib_port)
qp_attr.qkey = self.UD_QKEY
qp_attr.pkey_index = self.UD_PKEY_INDEX
self.qp = QP(self.pd, qp_init_attr, qp_attr)
def create_qp(self):
"""
Initializes self.qp with an RC QP.
:return: None
"""
qp_caps = QPCap(max_recv_wr=self.num_msgs)
qp_init_attr = QPInitAttr(qp_type=e.IBV_QPT_RC, scq=self.cq,
rcq=self.cq, cap=qp_caps)
qp_attr = QPAttr(port_num=self.ib_port)
self.qp = QP(self.pd, qp_init_attr, qp_attr)
def create_qp_cap(self):
"""
Create QPCap such that work queue elements will wrap around the send
work queue, this happens due to the iteration count being higher
than the max_send_wr.
:return:
"""
return QPCap(max_send_wr=4,
max_recv_wr=4,
max_recv_sge=2,
max_send_sge=2)
def create_qp(self):
qia = QPInitAttr(e.IBV_QPT_RAW_PACKET,
rcq=self.cq,
scq=self.cq,
cap=QPCap())
try:
qp = QP(self.pd, qia)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest(
"Create Raw Packet QP is not supported")
raise ex
qp.to_init(QPAttr())
return qp
def create_qp(self):
qp_caps = QPCap(max_recv_wr=self.num_msgs)
qp_init_attr = QPInitAttr(qp_type=e.IBV_QPT_RC,
scq=self.cq,
rcq=self.cq,
cap=qp_caps)
qp_attr = QPAttr(port_num=self.ib_port)
qp_access = e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_REMOTE_WRITE
qp_attr.qp_access_flags = qp_access
try:
self.qp = QP(self.pd, qp_init_attr, qp_attr)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create RC QP is not supported')
raise ex
def random_qp_cap(attr):
"""
Initializes a QPCap object with valid values based on the device's
attributes.
It doesn't check the max WR limits since they're reported for smaller WR
sizes.
:return: A QPCap object
"""
# We use significantly smaller values than those in device attributes.
# The attributes reported by the device don't take into account possible
# larger WQEs that include e.g. memory window.
send_wr = random.randint(1, int(attr.max_qp_wr / 8))
recv_wr = random.randint(1, int(attr.max_qp_wr / 8))
send_sge = random.randint(1, int(attr.max_sge / 2))
recv_sge = random.randint(1, int(attr.max_sge / 2))
return QPCap(send_wr, recv_wr, send_sge, recv_sge)
def __init__(self, addr=None, passive=None, **kwargs):
"""
:param addr: Local address to bind to.
:param passive: Indicate if this CM is the passive CM.
:param kwargs: Arguments:
* *port* (str)
Port number of the address
* *with_ext_qp* (bool)
If set, an external RC QP will be created and used by RDMACM
* *port_space* (str)
If set, indicates the CMIDs port space
"""
self.qp_init_attr = None
self.passive = passive
self.with_ext_qp = kwargs.get('with_ext_qp', False)
self.port = kwargs.get('port') if kwargs.get('port') else '7471'
self.port_space = kwargs.get('port_space', ce.RDMA_PS_TCP)
self.remote_operation = kwargs.get('remote_op')
self.qp_type = qp_type_per_ps[self.port_space]
self.qp_init_attr = QPInitAttr(qp_type=self.qp_type, cap=QPCap())
self.connected = False
# When passive side (server) listens to incoming connection requests,
# for each new request it creates a new cmid which is used to establish
# the connection with the remote side
self.msg_size = 1024
self.num_msgs = 10
self.channel = None
self.cq = None
self.qps = {}
self.mr = None
self.remote_qpn = None
self.ud_params = None
self.child_ids = {}
self.cmids = {}
if self.passive:
self.ai = AddrInfo(src=addr,
src_service=self.port,
port_space=self.port_space,
flags=ce.RAI_PASSIVE)
else:
self.ai = AddrInfo(src=addr,
dst=addr,
dst_service=self.port,
port_space=self.port_space)
def create_qp_init_attr(self):
self.qp_init_attr = QPInitAttr(cap=QPCap(max_recv_wr=1))
def create_qp_init_attr(rcq=None, scq=None):
return QPInitAttr(qp_type=e.IBV_QPT_RC,
rcq=rcq,
scq=scq,
cap=QPCap(max_recv_wr=1))
def create_qp(self):
qia = QPInitAttr(e.IBV_QPT_RAW_PACKET, rcq=self.cq, scq=self.cq,
cap=QPCap())
qp = QP(self.pd, qia)
qp.to_init(QPAttr())
return qp
def create_qp_cap(self):
return QPCap(max_send_wr=0, max_send_sge=0, max_recv_wr=0, max_recv_sge=0) if self.with_srq \
else QPCap(max_send_wr=4, max_send_sge=1, max_recv_wr=self.num_msgs, max_recv_sge=4)
def create_qp_cap(self):
return QPCap(self.num_msgs, 0, 1, 0)
def create_qp_init_attr():
return QPInitAttr(qp_type=e.IBV_QPT_RC, cap=QPCap(max_recv_wr=1))
def create_qp_cap(self):
return QPCap(max_recv_wr=self.num_msgs)
def create_qp_cap(self):
return QPCap(100, 0, 1, 0)
def create_qp_init_attr(self, rcq=None, scq=None):
return QPInitAttr(qp_type=self.qp_type, rcq=rcq, scq=scq,
cap=QPCap(max_recv_wr=1))
def create_qp_cap(self):
return QPCap(max_send_wr=self.num_msgs,
max_recv_wr=self.num_msgs,
max_inline_data=self.max_inline_data)