def test_create_qp_with_attr(self):
"""
Test QP creation via ibv_create_qp with a QPAttr object proivded.
Checked QP types are Raw Packet and UD. Raw Packet is skipped for
non-root users / Infiniband link layer.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
for i in range(1, attr.phys_port_cnt + 1):
qpts = [e.IBV_QPT_UD, e.IBV_QPT_RAW_PACKET] \
if u.is_eth(ctx, i) else [e.IBV_QPT_UD]
qia = get_qp_init_attr(cq, attr)
qia.qp_type = e.IBV_QPT_UD
with QP(pd, qia, QPAttr()) as qp:
assert qp.qp_state == e.IBV_QPS_RTS, 'UD QP should have been in RTS'
if u.is_eth(ctx, i) and u.is_root():
qia.qp_type = e.IBV_QPT_RAW_PACKET
with QP(pd, qia, QPAttr()) as qp:
assert qp.qp_state == e.IBV_QPS_RTS, 'Raw Packet QP should have been in RTS'
def test_dm_bad_access(self):
"""
Test multiple types of bad access to the Device Memory. Device memory
access requests a 4B alignment. The test tries to access the DM
with bad alignment or outside of the allocated memory.
"""
dm_size = 100
with d.DM(self.ctx, d.AllocDmAttr(length=dm_size)) as dm:
dm_access = e.IBV_ACCESS_ZERO_BASED | e.IBV_ACCESS_LOCAL_WRITE
dmmr = DMMR(PD(self.ctx), dm_size, dm_access, dm, 0)
access_cases = [(DM_INVALID_ALIGNMENT, 4), # Valid length with unaligned offset
(4, DM_INVALID_ALIGNMENT), # Valid offset with unaligned length
(dm_size + 4, 4), # Offset out of allocated memory
(0, dm_size + 4)] # Length out of allocated memory
for case in access_cases:
offset, length = case
with self.assertRaisesRegex(PyverbsRDMAError, 'Failed to copy from dm'):
dmmr.read(offset=offset, length=length)
with self.assertRaisesRegex(PyverbsRDMAError, 'Failed to copy to dm'):
dmmr.write(data='s'*length, offset=offset, length=length)
def test_reg_mr_bad_flags(self):
"""
Verify that illegal flags combination fails as expected
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
for i in range(5):
flags = random.sample([
e.IBV_ACCESS_REMOTE_WRITE, e.IBV_ACCESS_REMOTE_ATOMIC
], random.randint(1, 2))
mr_flags = 0
for i in flags:
mr_flags += i.value
try:
MR(pd, u.get_mr_length(), mr_flags)
except PyverbsRDMAError as err:
assert 'Failed to register a MR' in err.args[0]
else:
raise PyverbsRDMAError(
'Registered a MR with illegal falgs')
def test_modify_qp(self):
"""
Queries a QP after calling modify(). Verifies that its properties are
as expected.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
# Extended QP
qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex,
e.IBV_QPT_UD)
qp = QP(ctx, qia)
qa = QPAttr()
qa.qkey = 0x123
qp.to_init(qa)
qp_attr, qp_iattr = qp.query(e.IBV_QP_QKEY)
assert qp_attr.qkey == qa.qkey, 'Extended QP, QKey is not as expected'
qp.to_rtr(qa)
qa.sq_psn = 0x45
qp.to_rts(qa)
qp_attr, qp_iattr = qp.query(e.IBV_QP_SQ_PSN)
assert qp_attr.sq_psn == qa.sq_psn, 'Extended QP, SQ PSN is not as expected'
qa.qp_state = e.IBV_QPS_RESET
qp.modify(qa, e.IBV_QP_STATE)
assert qp.qp_state == e.IBV_QPS_RESET, 'Extended QP, QP state is not as expected'
# Legacy QP
qia = get_qp_init_attr(cq, attr)
qp = QP(pd, qia)
qa = QPAttr()
qa.qkey = 0x123
qp.to_init(qa)
qp_attr, qp_iattr = qp.query(e.IBV_QP_QKEY)
assert qp_attr.qkey == qa.qkey, 'Legacy QP, QKey is not as expected'
qp.to_rtr(qa)
qa.sq_psn = 0x45
qp.to_rts(qa)
qp_attr, qp_iattr = qp.query(e.IBV_QP_SQ_PSN)
assert qp_attr.sq_psn == qa.sq_psn, 'Legacy QP, SQ PSN is not as expected'
qa.qp_state = e.IBV_QPS_RESET
qp.modify(qa, e.IBV_QP_STATE)
assert qp.qp_state == e.IBV_QPS_RESET, 'Legacy QP, QP state is not as expected'
def test_modify_ud_qp(self):
"""
Queries a UD QP after calling modify(). Verifies that its properties are
as expected.
"""
with PD(self.ctx) as pd:
with CQ(self.ctx, 100, None, None, 0) as cq:
# Legacy QP
qia = u.get_qp_init_attr(cq, self.attr)
qia.qp_type = e.IBV_QPT_UD
qp = self.create_qp(pd, qia, False, False, self.ib_port)
qa = QPAttr()
qa.qkey = 0x123
qp.to_init(qa)
qp_attr, _ = qp.query(e.IBV_QP_QKEY)
assert qp_attr.qkey == qa.qkey, 'Legacy QP, QKey is not as expected'
qp.to_rtr(qa)
qa.sq_psn = 0x45
qp.to_rts(qa)
qp_attr, _ = qp.query(e.IBV_QP_SQ_PSN)
assert qp_attr.sq_psn == qa.sq_psn, 'Legacy QP, SQ PSN is not as expected'
qa.qp_state = e.IBV_QPS_RESET
qp.modify(qa, e.IBV_QP_STATE)
assert qp.qp_state == e.IBV_QPS_RESET, 'Legacy QP, QP state is not as expected'
# Extended QP
qia = get_qp_init_attr_ex(cq, pd, self.attr, self.attr_ex,
e.IBV_QPT_UD)
qp = self.create_qp(self.ctx, qia, True, False, self.ib_port)
qa = QPAttr()
qa.qkey = 0x123
qp.to_init(qa)
qp_attr, _ = qp.query(e.IBV_QP_QKEY)
assert qp_attr.qkey == qa.qkey, 'Extended QP, QKey is not as expected'
qp.to_rtr(qa)
qa.sq_psn = 0x45
qp.to_rts(qa)
qp_attr, _ = qp.query(e.IBV_QP_SQ_PSN)
assert qp_attr.sq_psn == qa.sq_psn, 'Extended QP, SQ PSN is not as expected'
qa.qp_state = e.IBV_QPS_RESET
qp.modify(qa, e.IBV_QP_STATE)
assert qp.qp_state == e.IBV_QPS_RESET, 'Extended QP, QP state is not as expected'
def test_dmabuf_reg_mr_bad_flags(self):
"""
Verify that illegal flags combination fails as expected
"""
check_dmabuf_support(self.gpu)
with PD(self.ctx) as pd:
check_dmabuf_mr_support(pd, self.gpu)
for i in range(5):
flags = random.sample([e.IBV_ACCESS_REMOTE_WRITE,
e.IBV_ACCESS_REMOTE_ATOMIC],
random.randint(1, 2))
mr_flags = 0
for i in flags:
mr_flags += i.value
try:
DmaBufMR(pd, u.get_mr_length(), mr_flags,
unit=self.gpu, gtt=self.gtt)
except PyverbsRDMAError as err:
assert 'Failed to register a dma-buf MR' in err.args[0]
else:
raise PyverbsRDMAError('Registered a dma-buf MR with illegal falgs')
def test_query_qp(self):
"""
Queries a QP after creation. Verifies that its properties are as
expected.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
for i in range(1, attr.phys_port_cnt + 1):
qpts = get_qp_types(ctx, i)
is_ex = random.choice([True, False])
if is_ex:
qia = get_qp_init_attr_ex(cq, pd, qpts, attr,
attr_ex)
else:
qia = get_qp_init_attr(cq, qpts, attr)
caps = qia.cap # Save them to verify values later
qp = QP(ctx, qia) if is_ex else QP(pd, qia)
attr, init_attr = qp.query(e.IBV_QP_CUR_STATE
| e.IBV_QP_CAP)
verify_qp_attrs(caps, e.IBV_QPS_RESET, init_attr, attr)
def test_mr_rereg_pd(self):
"""
Test that cover rereg MR's PD with this flow:
Use MR with QP that was created with the same PD. Then rereg the MR's PD
and use the MR with the same QP, expect the traffic to fail with "remote
operation error". Restate the QP from ERR state, rereg the MR back
to its previous PD and use it again with the QP, verify that it now
succeeds.
"""
self.create_players(MRRes)
u.traffic(**self.traffic_args)
server_new_pd = PD(self.server.ctx)
self.server.rereg_mr(flags=e.IBV_REREG_MR_CHANGE_PD, pd=server_new_pd)
with self.assertRaisesRegex(PyverbsRDMAError, 'Remote operation error'):
u.traffic(**self.traffic_args)
self.restate_qps()
self.server.rereg_mr(flags=e.IBV_REREG_MR_CHANGE_PD, pd=self.server.pd)
u.traffic(**self.traffic_args)
# Rereg the MR again with the new PD to cover
# destroying a PD with a re-registered MR.
self.server.rereg_mr(flags=e.IBV_REREG_MR_CHANGE_PD, pd=server_new_pd)
def test_create_dm_mr(self):
"""
Test ibv_reg_dm_mr
"""
for ctx, attr, attr_ex in self.devices:
if attr_ex.max_dm_size == 0:
raise unittest.SkipTest('Device memory is not supported')
with PD(ctx) as pd:
for i in range(10):
dm_len = random.randrange(u.MIN_DM_SIZE,
attr_ex.max_dm_size / 2,
u.DM_ALIGNMENT)
dm_attrs = u.get_dm_attrs(dm_len)
with d.DM(ctx, dm_attrs) as dm:
dm_mr_len = random.randint(1, dm_len)
dm_mr_offset = random.randint(0, (dm_len - dm_mr_len))
DMMR(pd,
dm_mr_len,
e.IBV_ACCESS_ZERO_BASED,
dm=dm,
offset=dm_mr_offset)
def query_qp_common_test(self, qp_type):
with PD(self.ctx) as pd:
with CQ(self.ctx, 100, None, None, 0) as cq:
if qp_type == e.IBV_QPT_RAW_PACKET:
if not (u.is_eth(self.ctx, self.ib_port) and u.is_root()):
raise unittest.SkipTest('To Create RAW QP must be done by root on Ethernet link layer')
# Legacy QP
qia = u.get_qp_init_attr(cq, self.attr)
qia.qp_type = qp_type
caps = qia.cap
qp = self.create_qp(pd, qia, False, False, self.ib_port)
qp_attr, qp_init_attr = qp.query(e.IBV_QP_STATE | e.IBV_QP_CAP)
self.verify_qp_attrs(caps, e.IBV_QPS_RESET, qp_init_attr, qp_attr)
# Extended QP
qia = get_qp_init_attr_ex(cq, pd, self.attr, self.attr_ex, qp_type)
caps = qia.cap # Save them to verify values later
qp = self.create_qp(self.ctx, qia, True, False, self.ib_port)
qp_attr, qp_init_attr = qp.query(e.IBV_QP_STATE | e.IBV_QP_CAP)
self.verify_qp_attrs(caps, e.IBV_QPS_RESET, qp_init_attr, qp_attr)
def test_destroy_dm_mr(self):
"""
Test freeing of dm_mr
"""
for ctx, attr, attr_ex in self.devices:
if attr_ex.max_dm_size == 0:
return
with PD(ctx) as pd:
for i in range(10):
dm_len = random.randrange(u.MIN_DM_SIZE,
attr_ex.max_dm_size / 2,
u.DM_ALIGNMENT)
dm_attrs = u.get_dm_attrs(dm_len)
with d.DM(ctx, dm_attrs) as dm:
dm_mr_len = random.randint(1, dm_len)
dm_mr_offset = random.randint(0, (dm_len - dm_mr_len))
dm_mr = DMMR(pd,
dm_mr_len,
e.IBV_ACCESS_ZERO_BASED,
dm=dm,
offset=dm_mr_offset)
dm_mr.close()
def create_resources(self, bad_flow_type=0, **resource_arg):
"""
Creates DmaGga test resources that include a "server" resource that
can be used to send the MEMCPY WR, and a destination MR to copy data to.
The destination MR can be created on a different PD or with insufficient
access permissions, according to the bad_flow_type.
:param bad_flow_type: (Optional) An enum of BadFlowType that indicates
the bad flow type (default: 0 - good flow)
:param resource_arg: Dict of args that specify the resource specific
attributes.
:return: None
"""
self.server = Mlx5DmaResources(**self.dev_info, **resource_arg)
self.dest_pd = self.server.pd
dest_mr_access = e.IBV_ACCESS_LOCAL_WRITE
if bad_flow_type == BadFlowType.DIFFERENT_PD:
self.dest_pd = PD(self.server.ctx)
elif bad_flow_type == BadFlowType.MR_ILLEGAL_ACCESS:
dest_mr_access = e.IBV_ACCESS_REMOTE_READ
self.dest_mr = MR(self.dest_pd, self.server.msg_size, dest_mr_access)
# No need to connect the QPs
self.server.pre_run([0], [0])
def test_dmabuf_dereg_mr_twice(self):
"""
Verify that explicit call to DmaBufMR's close() doesn't fail
"""
check_dmabuf_support(self.gpu)
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
check_dmabuf_mr_support(pd, self.gpu)
flags = u.get_dmabuf_access_flags(ctx)
for f in flags:
len = u.get_mr_length()
for off in [0, len // 2]:
with DmaBufMR(pd,
len,
f,
offset=off,
gpu=self.gpu,
gtt=self.gtt) as mr:
# Pyverbs supports multiple destruction of objects,
# we are not expecting an exception here.
mr.close()
mr.close()
def test_dmabuf_write(self):
"""
Test writing to DmaBufMR's buffer
"""
check_dmabuf_support(self.gpu)
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
check_dmabuf_mr_support(pd, self.gpu)
for i in range(10):
mr_len = u.get_mr_length()
flags = u.get_dmabuf_access_flags(ctx)
for f in flags:
for mr_off in [0, mr_len // 2]:
with DmaBufMR(pd,
mr_len,
f,
offset=mr_off,
gpu=self.gpu,
gtt=self.gtt) as mr:
write_len = min(random.randint(1, MAX_IO_LEN),
mr_len)
mr.write('a' * write_len, write_len)
def create_qp_common_test(self, qp_type, qp_state, is_ex, with_attr):
"""
Common function used by create QP tests.
"""
with PD(self.ctx) as pd:
with CQ(self.ctx, 100, None, None, 0) as cq:
if qp_type == e.IBV_QPT_RAW_PACKET:
if not (u.is_eth(self.ctx, self.ib_port) and u.is_root()):
raise unittest.SkipTest('To Create RAW QP must be done by root on Ethernet link layer')
if is_ex:
qia = get_qp_init_attr_ex(cq, pd, self.attr, self.attr_ex, qp_type)
creator = self.ctx
else:
qia = u.get_qp_init_attr(cq, self.attr)
qia.qp_type = qp_type
creator = pd
qp = self.create_qp(creator, qia, is_ex, with_attr, self.ib_port)
qp_type_str = pu.qp_type_to_str(qp_type)
qp_state_str = pu.qp_state_to_str(qp_state)
assert qp.qp_state == qp_state , f'{qp_type_str} QP should have been in {qp_state_str}'
def test_dmabuf_read(self):
"""
Test reading from DmaBufMR's buffer
"""
check_dmabuf_support(self.gpu)
with PD(self.ctx) as pd:
check_dmabuf_mr_support(pd, self.gpu)
for i in range(10):
mr_len = u.get_mr_length()
flags = u.get_dmabuf_access_flags(self.ctx)
for f in flags:
for mr_off in [0, mr_len//2]:
with DmaBufMR(pd, mr_len, f, offset=mr_off,
unit=self.gpu, gtt=self.gtt) as mr:
write_len = min(random.randint(1, MAX_IO_LEN),
mr_len)
write_str = 'a' * write_len
mr.write(write_str, write_len)
read_len = random.randint(1, write_len)
offset = random.randint(0, write_len-read_len)
read_str = mr.read(read_len, offset).decode()
assert read_str in write_str
def test_create_dm_mr(self):
"""
Test ibv_reg_dm_mr
"""
lst = d.get_device_list()
for dev in lst:
with d.Context(name=dev.name.decode()) as ctx:
attr = ctx.query_device_ex()
if attr.max_dm_size == 0:
return
with PD(ctx) as pd:
dm_len = random.randrange(u.MIN_DM_SIZE, attr.max_dm_size,
u.DM_ALIGNMENT)
dm_attrs = u.get_dm_attrs(dm_len)
with d.DM(ctx, dm_attrs) as dm:
dm_mr_len = random.randint(1, dm_len)
dm_mr_offset = random.randint(0, (dm_len - dm_mr_len))
dm_mr = DMMR(pd,
dm_mr_len,
e.IBV_ACCESS_ZERO_BASED,
dm=dm,
offset=dm_mr_offset)
def test_create_ah(self):
"""
Test ibv_create_ah.
"""
done = 0
for ctx, attr, attr_ex in self.devices:
pd = PD(ctx)
for port_num in range(1, 1 + attr.phys_port_cnt):
state = ctx.query_port(port_num).state
if state != e.IBV_PORT_ACTIVE and state != e.IBV_PORT_INIT:
continue
gr = u.get_global_route(ctx, port_num=port_num)
ah_attr = AHAttr(gr=gr, is_global=1, port_num=port_num)
try:
with AH(pd, attr=ah_attr):
done += 1
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create AH is not supported')
raise ex
if done == 0:
raise unittest.SkipTest('No port is up, can\'t create AH')
def test_create_qp_ex_no_attr_connected(self):
"""
Test QP creation via ibv_create_qp_ex without a QPAttr object proivded.
Checked QP types are RC and UC.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex, e.IBV_QPT_RC)
try:
with QP(ctx, qia) as qp:
assert qp.qp_state == e.IBV_QPS_RESET, 'RC QP should have been in RESET'
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create QP with extended attrs is not supported')
raise ex
qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex, e.IBV_QPT_UC)
try:
with QP(ctx, qia) as qp:
assert qp.qp_state == e.IBV_QPS_RESET, 'UC QP should have been in RESET'
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create QP with extended attrs is not supported')
raise ex
def test_query_qp(self):
"""
Queries a QP after creation. Verifies that its properties are as
expected.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
for i in range(1, attr.phys_port_cnt + 1):
qpts = get_qp_types(ctx, i)
for qpt in qpts:
# Extended QP
qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex,
qpt)
caps = qia.cap # Save them to verify values later
try:
qp = QP(ctx, qia)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest(
'Create QP with extended attrs is not supported'
)
raise ex
qp_attr, qp_init_attr = qp.query(e.IBV_QP_STATE
| e.IBV_QP_CAP)
verify_qp_attrs(caps, e.IBV_QPS_RESET,
qp_init_attr, qp_attr)
# Legacy QP
qia = get_qp_init_attr(cq, attr)
qia.qp_type = qpt
caps = qia.cap # Save them to verify values later
qp = QP(pd, qia)
qp_attr, qp_init_attr = qp.query(e.IBV_QP_STATE
| e.IBV_QP_CAP)
verify_qp_attrs(caps, e.IBV_QPS_RESET,
qp_init_attr, qp_attr)
def test_dealloc_pd(self):
"""
Test ibv_dealloc_pd()
"""
with PD(self.ctx) as pd:
pd.close()
def test_dereg_mw_type2(self):
""" Test ibv_dealloc_mw() """
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with MW(pd, e.IBV_MW_TYPE_2) as mw:
mw.close()
def test_reg_mw_type2(self):
""" Test ibv_alloc_mw() """
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with MW(pd, e.IBV_MW_TYPE_2):
pass
def create_pd(self):
self.pd = PD(self.ctx)
self.dv_pd = Mlx5DvObj(dve.MLX5DV_OBJ_PD, pd=self.pd).dvpd
def test_alloc_pd(self):
"""
Test ibv_alloc_pd()
"""
with PD(self.ctx):
pass
def create_pd(self):
self.pd = PD(self.ctx)
def create_pd(self):
self.pd = PD(self.ctx, handle=self.pd_handle)
