def test_create_qp_with_attr_connected(self):
"""
Test QP creation via ibv_create_qp without a QPAttr object proivded.
QP type can be either RC or UC.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
qia = get_qp_init_attr(cq, [e.IBV_QPT_RC, e.IBV_QPT_UC],
attr)
with QP(pd, qia, QPAttr()) as qp:
assert qp.qp_state == e.IBV_QPS_INIT
def test_create_qp_ex_with_attr(self):
"""
Test QP creation via ibv_create_qp_ex with a QPAttr object proivded.
Checked QP types are Raw Packet and UD. Raw Packet is skipped for
non-root users / Infiniband link layer.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
for i in range(1, attr.phys_port_cnt + 1):
qpts = [e.IBV_QPT_UD, e.IBV_QPT_RAW_PACKET] \
if is_eth(ctx, i) else [e.IBV_QPT_UD]
qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex,
e.IBV_QPT_UD)
with QP(ctx, qia, QPAttr()) as qp:
assert qp.qp_state == e.IBV_QPS_RTS, 'UD QP should have been in RTS'
if is_eth(ctx, i) and is_root():
qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex,
e.IBV_QPT_RAW_PACKET)
with QP(ctx, qia, QPAttr()) as qp:
assert qp.qp_state == e.IBV_QPS_RTS, 'Raw Packet QP should have been in RTS'
def create_qp(self):
"""
Initializes self.qp with an RC QP.
:return: None
"""
qp_caps = QPCap(max_recv_wr=self.num_msgs)
qp_init_attr = QPInitAttr(qp_type=e.IBV_QPT_RC,
scq=self.cq,
rcq=self.cq,
cap=qp_caps)
qp_attr = QPAttr(port_num=self.ib_port)
self.qp = QP(self.pd, qp_init_attr, qp_attr)
def test_modify_ud_qp(self):
"""
Queries a UD QP after calling modify(). Verifies that its properties are
as expected.
"""
with PD(self.ctx) as pd:
with CQ(self.ctx, 100, None, None, 0) as cq:
# Legacy QP
qia = u.get_qp_init_attr(cq, self.attr)
qia.qp_type = e.IBV_QPT_UD
qp = self.create_qp(pd, qia, False, False, self.ib_port)
qa = QPAttr()
qa.qkey = 0x123
qp.to_init(qa)
qp_attr, _ = qp.query(e.IBV_QP_QKEY)
assert qp_attr.qkey == qa.qkey, 'Legacy QP, QKey is not as expected'
qp.to_rtr(qa)
qa.sq_psn = 0x45
qp.to_rts(qa)
qp_attr, _ = qp.query(e.IBV_QP_SQ_PSN)
assert qp_attr.sq_psn == qa.sq_psn, 'Legacy QP, SQ PSN is not as expected'
qa.qp_state = e.IBV_QPS_RESET
qp.modify(qa, e.IBV_QP_STATE)
assert qp.qp_state == e.IBV_QPS_RESET, 'Legacy QP, QP state is not as expected'
# Extended QP
qia = get_qp_init_attr_ex(cq, pd, self.attr, self.attr_ex, e.IBV_QPT_UD)
qp = self.create_qp(self.ctx, qia, True, False, self.ib_port)
qa = QPAttr()
qa.qkey = 0x123
qp.to_init(qa)
qp_attr, _ = qp.query(e.IBV_QP_QKEY)
assert qp_attr.qkey == qa.qkey, 'Extended QP, QKey is not as expected'
qp.to_rtr(qa)
qa.sq_psn = 0x45
qp.to_rts(qa)
qp_attr, _ = qp.query(e.IBV_QP_SQ_PSN)
assert qp_attr.sq_psn == qa.sq_psn, 'Extended QP, SQ PSN is not as expected'
qa.qp_state = e.IBV_QPS_RESET
qp.modify(qa, e.IBV_QP_STATE)
assert qp.qp_state == e.IBV_QPS_RESET, 'Extended QP, QP state is not as expected'
def create_qp(self, creator, qp_init_attr, is_ex, with_attr, port_num):
"""
Auxiliary function to create QP object.
"""
try:
qp_attr = (None, QPAttr(port_num=port_num))[with_attr]
return QP(creator, qp_init_attr, qp_attr)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
with_str = ('without', 'with')[with_attr] + ('', ' extended')[is_ex]
qp_type_str = pu.qp_type_to_str(qp_init_attr.qp_type)
raise unittest.SkipTest(f'Create {qp_type_str} QP {with_str} attrs is not supported')
raise ex
def reset_qp(self, qp_idx):
qp_attr = QPAttr(qp_state=e.IBV_QPS_RESET)
self.qps[qp_idx].modify(qp_attr, e.IBV_QP_STATE)
self.qps[qp_idx].to_rts(qp_attr)
err_mask = self.qp_stream_errors[qp_idx][0]
index = 0
# Clear all set dci
while err_mask != 0:
if (err_mask & 0x1) == 0x1:
Mlx5QP.modify_dci_stream_channel_id(self.qps[qp_idx], index)
index += 1
err_mask >>= 1
self.qp_stream_errors[qp_idx][0] = 0
def create_qp(self):
"""
Create an rdmacm QP. If self.with_ext_qp is set, then an external CQ and
RC QP will be created and set in self.cq and self.qp
respectively.
"""
cmid = self.child_id if self.passive else self.cmid
if not self.with_ext_qp:
cmid.create_qp(self.create_qp_init_attr())
else:
self.cq = CQ(cmid.context, self.num_msgs, None, None, 0)
init_attr = self.create_qp_init_attr(rcq=self.cq, scq=self.cq)
self.qp = QP(cmid.pd, init_attr, QPAttr())
def test_create_qp_with_attr(self):
"""
Test QP creation via ibv_create_qp with a QPAttr object proivded.
QP type can be either Raw Packet or UD.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
for i in range(1, attr.phys_port_cnt + 1):
qpts = [e.IBV_QPT_UD, e.IBV_QPT_RAW_PACKET] \
if is_eth(ctx, i) else [e.IBV_QPT_UD]
qia = get_qp_init_attr(cq, qpts, attr)
with QP(pd, qia, QPAttr()) as qp:
assert qp.qp_state == e.IBV_QPS_RTS
def create_qp(self, conn_idx=0):
"""
Create an rdmacm QP. If self.with_ext_qp is set, then an external CQ and
QP will be created. In case that CQ is already created, it is used
for the newly created QP.
:param conn_idx: The connection index.
"""
cmid = self.child_id if self.passive else self.cmid
if not self.with_ext_qp:
cmid.create_qp(self.create_qp_init_attr())
else:
self.create_cq(cmid)
init_attr = self.create_qp_init_attr(rcq=self.cq, scq=self.cq)
self.qps[conn_idx] = QP(cmid.pd, init_attr, QPAttr())
def create_qp(self):
qia = QPInitAttr(e.IBV_QPT_RAW_PACKET,
rcq=self.cq,
scq=self.cq,
cap=QPCap())
try:
qp = QP(self.pd, qia)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest(
"Create Raw Packet QP is not supported")
raise ex
qp.to_init(QPAttr())
return qp
def to_rts(self):
gid = self.ctx.query_gid(self.ib_port, self.gid_index)
gr = GlobalRoute(dgid=gid, sgid_index=self.gid_index)
ah_attr = AHAttr(port_num=self.ib_port, is_global=True,
gr=gr, dlid=self.port_attr.lid)
qp_attr = QPAttr()
qp_attr.path_mtu = PATH_MTU
set_rnr_attributes(qp_attr)
qp_attr.ah_attr = ah_attr
for i in range(self.qp_count):
qp_attr.dest_qp_num = self.rqps_num[i][1]
qp_attr.rq_psn = self.psns[i]
qp_attr.sq_psn = self.rpsns[i]
self.rqp_lst[i].to_rts(qp_attr)
qp_attr.dest_qp_num = self.rqps_num[i][0]
self.sqp_lst[i].to_rts(qp_attr)
def create_qp_ex(agr_obj, qp_type, send_flags):
if qp_type == e.IBV_QPT_XRC_SEND:
cap = QPCap(max_send_wr=agr_obj.num_msgs,
max_recv_wr=0,
max_recv_sge=0,
max_send_sge=1)
else:
cap = QPCap(max_send_wr=agr_obj.num_msgs,
max_recv_wr=agr_obj.num_msgs,
max_recv_sge=1,
max_send_sge=1)
qia = QPInitAttrEx(cap=cap,
qp_type=qp_type,
scq=agr_obj.cq,
rcq=agr_obj.cq,
pd=agr_obj.pd,
send_ops_flags=send_flags,
comp_mask=e.IBV_QP_INIT_ATTR_PD
| e.IBV_QP_INIT_ATTR_SEND_OPS_FLAGS)
qp_attr = QPAttr(port_num=agr_obj.ib_port)
if qp_type == e.IBV_QPT_UD:
qp_attr.qkey = agr_obj.UD_QKEY
qp_attr.pkey_index = agr_obj.UD_PKEY_INDEX
if qp_type == e.IBV_QPT_RC:
qp_attr.qp_access_flags = e.IBV_ACCESS_REMOTE_WRITE | \
e.IBV_ACCESS_REMOTE_READ | \
e.IBV_ACCESS_REMOTE_ATOMIC
try:
# We don't have capability bits for this
qp = QPEx(agr_obj.ctx, qia, qp_attr)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest(
'Extended QP is not supported on this device')
raise ex
return qp
def test_mkey_sig_pipelining_bad(self):
"""
Test the bad signature pipelining scenario.
"""
self.create_players(
Mlx5MkeyResources,
dv_send_ops_flags=dve.MLX5DV_QP_EX_WITH_MKEY_CONFIGURE,
mkey_create_flags=dve.MLX5DV_MKEY_INIT_ATTR_FLAGS_INDIRECT
| dve.MLX5DV_MKEY_INIT_ATTR_FLAGS_BLOCK_SIGNATURE,
dv_qp_create_flags=dve.MLX5DV_QP_CREATE_DISABLE_SCATTER_TO_CQE
| dve.MLX5DV_QP_CREATE_SIG_PIPELINING)
self.reg_mr_sig_pipelining_client(check_mask=dve.MLX5DV_SIG_MASK_CRC32)
self.reg_mr_sig_pipelining_server()
self.build_traffic_elements_sig_pipelining()
self.server.qp.post_recv(self.server_resp_wr)
self.client.qp.post_send(self.client_data_wr)
self.client.qp.post_send(self.client_resp_wr)
# Expect SQ_DRAINED event
event = self.client.ctx.get_async_event()
event.ack()
self.assertEqual(event.event_type, e.IBV_EVENT_SQ_DRAINED)
# No completion is expected on the client side
nc, _ = self.client.cq.poll(1)
self.assertEqual(nc, 0)
# No completion is expected on the server side
nc, _ = self.server.cq.poll(1)
self.assertEqual(nc, 0)
self.check_mkey(self.client, dve.MLX5DV_MKEY_SIG_BLOCK_BAD_GUARD)
self.check_mkey(self.server)
# Cancel and repost response WR
canceled_count = self.client.qp.cancel_posted_send_wrs(1)
self.assertEqual(canceled_count, 1)
self.client.qp.post_send(self.client_resp_wr)
# Move QP back to RTS and receive completions
self.client.qp.modify(
QPAttr(qp_state=e.IBV_QPS_RTS, cur_qp_state=e.IBV_QPS_SQD),
e.IBV_QP_STATE | e.IBV_QP_CUR_STATE)
u.poll_cq(self.client.cq)
u.poll_cq(self.server.cq)
def create_qp(self):
qia = QPInitAttr(e.IBV_QPT_RAW_PACKET, rcq=self.cq, scq=self.cq,
cap=QPCap())
qp = QP(self.pd, qia)
qp.to_init(QPAttr())
return qp
def to_rts(self):
gid = self.ctx.query_gid(self.ib_port, self.gid_index)
gr = GlobalRoute(dgid=gid, sgid_index=self.gid_index)
ah_attr = AHAttr(port_num=self.ib_port, is_global=True,
gr=gr, dlid=self.port_attr.lid)
qp_attr = QPAttr()
qp_attr.path_mtu = PATH_MTU
qp_attr.timeout = TIMEOUT
qp_attr.retry_cnt = RETRY_CNT
qp_attr.rnr_retry = RNR_RETRY
qp_attr.min_rnr_timer = MIN_RNR_TIMER
qp_attr.ah_attr = ah_attr
for i in range(self.qp_count):
qp_attr.dest_qp_num = self.rqps_num[i][1]
qp_attr.rq_psn = self.psns[i]
qp_attr.sq_psn = self.rpsns[i]
self.rqp_lst[i].to_rts(qp_attr)
qp_attr.dest_qp_num = self.rqps_num[i][0]
self.sqp_lst[i].to_rts(qp_attr)
def to_rts(self):
"""
Set the QP attributes' values to arbitrary values (same values used in
ibv_rc_pingpong).
:return: None
"""
attr = QPAttr(port_num=self.ib_port)
attr.dest_qp_num = self.rqpn
attr.path_mtu = PATH_MTU
attr.max_dest_rd_atomic = MAX_DEST_RD_ATOMIC
attr.min_rnr_timer = MIN_RNR_TIMER
attr.rq_psn = self.psn
attr.sq_psn = self.rpsn
attr.timeout = TIMEOUT
attr.retry_cnt = RETRY_CNT
attr.rnr_retry = RNR_RETRY
attr.max_rd_atomic = MAX_RD_ATOMIC
gr = GlobalRoute(dgid=self.ctx.query_gid(self.ib_port, self.gid_index),
sgid_index=self.gid_index)
ah_attr = AHAttr(port_num=self.ib_port, is_global=1, gr=gr,
dlid=self.port_attr.lid)
attr.ah_attr = ah_attr
self.qp.to_rts(attr)
def create_qp_attr(self):
qp_attr = QPAttr(port_num=self.ib_port)
qp_attr.qp_access_flags = self.qp_access
return qp_attr
def create_qp_attr(self):
attr = QPAttr(port_num=self.ib_port)
attr.qkey = self.SRD_QKEY
attr.pkey_index = self.SRD_PKEY_INDEX
return attr
def create_qp_attr(self):
return QPAttr(port_num=self.ib_port)
def create_qp_attr(self):
qp_attr = QPAttr(port_num=self.ib_port)
qp_attr.qp_access_flags = e.IBV_ACCESS_LOCAL_WRITE
if self.remote_access:
qp_attr.qp_access_flags |= e.IBV_ACCESS_REMOTE_WRITE
return qp_attr
def create_qp_attr(self):
qp_attr = QPAttr(port_num=self.ib_port)
qp_access = e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_REMOTE_WRITE
qp_attr.qp_access_flags = qp_access
return qp_attr
def reset_qp(self, qp_idx):
qp_attr = QPAttr(qp_state=e.IBV_QPS_RESET)
self.qps[qp_idx].modify(qp_attr, e.IBV_QP_STATE)
self.qps[qp_idx].to_rts(qp_attr)
self.qp_stream_errors[qp_idx][0] = 0
