class ImportResources(base_class):
def __init__(self,
cmd_fd,
pd_handle,
mr_handle,
mr_addr=None,
**kwargs):
self.cmd_fd = cmd_fd
self.pd_handle = pd_handle
self.mr_handle = mr_handle
self.mr_addr = mr_addr
super(ImportResources, self).__init__(**kwargs)
def create_context(self):
try:
self.ctx = Context(cmd_fd=self.cmd_fd)
except u.PyverbsRDMAError as ex:
if ex.error_code in [errno.EOPNOTSUPP, errno.EPROTONOSUPPORT]:
raise unittest.SkipTest(
'Importing a device is not supported')
raise ex
def create_pd(self):
self.pd = PD(self.ctx, handle=self.pd_handle)
def create_mr(self):
self.mr = MR(self.pd, handle=self.mr_handle, address=self.mr_addr)
def unimport_resources(self):
self.mr.unimport()
self.pd.unimport()
self.pd.close()
def test_modify_qp(self):
"""
Queries a QP after calling modify(). Verifies that its properties are
as expected.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
is_ex = random.choice([True, False])
if is_ex:
qia = get_qp_init_attr_ex(cq, pd, [e.IBV_QPT_UD], attr,
attr_ex)
else:
qia = get_qp_init_attr(cq, [e.IBV_QPT_UD], attr)
qp = QP(ctx, qia) if is_ex \
else QP(pd, qia)
qa = QPAttr()
qa.qkey = 0x123
qp.to_init(qa)
attr, iattr = qp.query(e.IBV_QP_QKEY)
assert attr.qkey == qa.qkey
qp.to_rtr(qa)
qa.sq_psn = 0x45
qp.to_rts(qa)
attr, iattr = qp.query(e.IBV_QP_SQ_PSN)
assert attr.sq_psn == qa.sq_psn
qa.qp_state = e.IBV_QPS_RESET
qp.modify(qa, e.IBV_QP_STATE)
assert qp.qp_state == e.IBV_QPS_RESET
def set_bad_flow(self, bad_flow):
self.bad_flow = bad_flow
if self.bad_flow:
self.pd_bad = PD(self.ctx)
self.mr_bad_flow = False
if bad_flow == DCI_TEST_BAD_FLOW_WITH_RESET:
self.stream_check = True
def test_create_qp_no_attr(self):
"""
Test QP creation via ibv_create_qp without a QPAttr object proivded.
Checked QP types are Raw Packet and UD. Raw Packet is skipped for
non-root users / Infiniband link layer.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
for i in range(1, attr.phys_port_cnt + 1):
qia = get_qp_init_attr(cq, attr)
qia.qp_type = e.IBV_QPT_UD
with QP(pd, qia) as qp:
assert qp.qp_state == e.IBV_QPS_RESET, 'UD QP should have been in RESET'
if u.is_eth(ctx, i) and u.is_root():
qia.qp_type = e.IBV_QPT_RAW_PACKET
try:
with QP(pd, qia) as qp:
assert qp.qp_state == e.IBV_QPS_RESET, 'Raw Packet QP should have been in RESET'
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest(
"Create Raw Packet QP is not supported"
)
raise ex
def test_create_qp_no_attr_connected(self):
"""
Test QP creation via ibv_create_qp without a QPAttr object proivded.
Checked QP types are RC and UC.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
qia = get_qp_init_attr(cq, attr)
qia.qp_type = e.IBV_QPT_RC
try:
with QP(pd, qia) as qp:
assert qp.qp_state == e.IBV_QPS_RESET, 'RC QP should have been in RESET'
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest(
'Create QP with RC attrs is not supported')
raise ex
qia.qp_type = e.IBV_QPT_UC
try:
with QP(pd, qia) as qp:
assert qp.qp_state == e.IBV_QPS_RESET, 'UC QP should have been in RESET'
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest(
'Create QP with UC attrs is not supported')
raise ex
def _create_mr_devx(self, rdma_device, buf_size):
"""This method create mkey and memory buffer using devx (part of ofed driver).
"""
mlx5dv_attr = Mlx5DVContextAttr()
self.ctx = Mlx5Context(mlx5dv_attr, name=rdma_device)
self.my_pd = PD(self.ctx)
self.dv_pd = Mlx5DvObj(dve.MLX5DV_OBJ_PD, pd=self.my_pd).dvpd
self.umem = self._create_umem(self.ctx, buf_size)
self.mkey_ctx = SwMkc(umr_en=0,
lr=1,
lw=1,
access_mode_1_0=0x1,
pd=self.dv_pd.pdn,
start_addr=self.umem.umem_addr,
len=buf_size)
mkey_in = CreateMkeyIn(sw_mkc=self.mkey_ctx,
mkey_umem_id=self.umem.umem_id,
mkey_umem_valid=1)
self.mkey_obj = Mlx5DevxObj(self.ctx, mkey_in,
len(CreateMkeyOut()))
mkey_out = CreateMkeyOut(self.mkey_obj.out_view)
if mkey_out.status:
raise PyverbsRDMAError(
"Failed to create mkey with syndrome {0}".format(
mkey_out.syndrome))
mkey_index = mkey_out.mkey_index
self.lkey = mkey_index << 8
def test_query_qp(self):
"""
Queries a QP after creation. Verifies that its properties are as
expected.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
for i in range(1, attr.phys_port_cnt + 1):
qpts = get_qp_types(ctx, i)
for qpt in qpts:
# Extended QP
qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex,
qpt)
caps = qia.cap # Save them to verify values later
qp = QP(ctx, qia)
qp_attr, qp_init_attr = qp.query(e.IBV_QP_CUR_STATE
| e.IBV_QP_CAP)
verify_qp_attrs(caps, e.IBV_QPS_RESET,
qp_init_attr, qp_attr)
# Legacy QP
qia = get_qp_init_attr(cq, attr)
qia.qp_type = qpt
caps = qia.cap # Save them to verify values later
qp = QP(pd, qia)
qp_attr, qp_init_attr = qp.query(e.IBV_QP_CUR_STATE
| e.IBV_QP_CAP)
verify_qp_attrs(caps, e.IBV_QPS_RESET,
qp_init_attr, qp_attr)
def test_query_qp(self):
"""
Queries a QP after creation. Verifies that its properties are as
expected.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
for i in range(1, attr.phys_port_cnt + 1):
qpts = get_qp_types(ctx, i)
for qpt in qpts:
# Extended QP
qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex,
qpt)
caps = qia.cap # Save them to verify values later
try:
qp = QP(ctx, qia)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create QP with extended attrs is not supported')
raise ex
qp_attr, qp_init_attr = qp.query(e.IBV_QP_CUR_STATE |
e.IBV_QP_CAP)
verify_qp_attrs(caps, e.IBV_QPS_RESET, qp_init_attr,
qp_attr)
# Legacy QP
qia = get_qp_init_attr(cq, attr)
qia.qp_type = qpt
caps = qia.cap # Save them to verify values later
qp = QP(pd, qia)
qp_attr, qp_init_attr = qp.query(e.IBV_QP_CUR_STATE |
e.IBV_QP_CAP)
verify_qp_attrs(caps, e.IBV_QPS_RESET, qp_init_attr,
qp_attr)
def test_multiple_pd_creation(self):
"""
Test multiple creations and destructions of a PD object
"""
for i in range(random.randint(1, 200)):
with PD(self.ctx) as pd:
pd.close()
def test_dmabuf_reg_mr_bad_flags(self):
"""
Verify that illegal flags combination fails as expected
"""
check_dmabuf_support(self.gpu)
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
check_dmabuf_mr_support(pd, self.gpu)
for i in range(5):
flags = random.sample([
e.IBV_ACCESS_REMOTE_WRITE, e.IBV_ACCESS_REMOTE_ATOMIC
], random.randint(1, 2))
mr_flags = 0
for i in flags:
mr_flags += i.value
try:
DmaBufMR(pd,
u.get_mr_length(),
mr_flags,
gpu=self.gpu,
gtt=self.gtt)
except PyverbsRDMAError as err:
assert 'Failed to register a dma-buf MR' in err.args[0]
else:
raise PyverbsRDMAError(
'Registered a dma-buf MR with illegal falgs')
def test_mlx5_dek_management(self):
"""
Test crypto login and DEK management APIs.
The test checks also that invalid actions are not permited, e.g, create
DEK not in login session.
"""
try:
self.pd = PD(self.ctx)
cred_bytes = struct.pack('!6Q', *self.crypto_details['credential'])
key = struct.pack('!5Q', *self.crypto_details['wrapped_key'])
self.dek_init_attr = \
Mlx5DEKInitAttr(self.pd, key=key,
key_size=dve.MLX5DV_CRYPTO_KEY_SIZE_128,
key_purpose=dve.MLX5DV_CRYPTO_KEY_PURPOSE_AES_XTS,
opaque=DEK_OPAQUE)
self.verify_create_dek_out_of_login_session()
self.verify_login_state(dve.MLX5DV_CRYPTO_LOGIN_STATE_NO_LOGIN)
# Login to crypto session
self.login_attr = Mlx5CryptoLoginAttr(cred_bytes)
Mlx5Context.crypto_login(self.ctx, self.login_attr)
self.verify_login_state(dve.MLX5DV_CRYPTO_LOGIN_STATE_VALID)
self.verify_login_twice()
self.dek = Mlx5DEK(self.ctx, self.dek_init_attr)
self.verify_dek_opaque()
self.dek.close()
# Logout from crypto session
Mlx5Context.crypto_logout(self.ctx)
self.verify_login_state(dve.MLX5DV_CRYPTO_LOGIN_STATE_NO_LOGIN)
except PyverbsRDMAError as ex:
print(ex)
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create crypto elements is not supported')
raise ex
def test_alloc_pd(self):
"""
Test ibv_alloc_pd()
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx):
pass
def test_create_qp_ex_with_attr_connected(self):
"""
Test QP creation via ibv_create_qp_ex with a QPAttr object proivded.
Checked QP type are RC and UC.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex,
e.IBV_QPT_RC)
try:
with QP(ctx, qia, QPAttr()) as qp:
assert qp.qp_state == e.IBV_QPS_INIT, 'RC QP should have been in INIT'
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create QP with extended attrs is not supported')
raise ex
qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex,
e.IBV_QPT_UC)
try:
with QP(ctx, qia, QPAttr()) as qp:
assert qp.qp_state == e.IBV_QPS_INIT, 'UC QP should have been in INIT'
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create QP with extended attrs is not supported')
raise ex
def test_dealloc_pd(self):
"""
Test ibv_dealloc_pd()
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
pd.close()
def query_qp_common_test(self, qp_type):
with PD(self.ctx) as pd:
with CQ(self.ctx, 100, None, None, 0) as cq:
if qp_type == e.IBV_QPT_RAW_PACKET:
if not (u.is_eth(self.ctx, self.ib_port) and u.is_root()):
raise unittest.SkipTest(
'To Create RAW QP must be done by root on Ethernet link layer'
)
# Legacy QP
qia = u.get_qp_init_attr(cq, self.attr)
qia.qp_type = qp_type
caps = qia.cap
qp = self.create_qp(pd, qia, False, False, self.ib_port)
qp_attr, qp_init_attr = qp.query(e.IBV_QP_STATE | e.IBV_QP_CAP)
self.verify_qp_attrs(caps, e.IBV_QPS_RESET, qp_init_attr,
qp_attr)
# Extended QP
qia = get_qp_init_attr_ex(cq, pd, self.attr, self.attr_ex,
qp_type)
caps = qia.cap # Save them to verify values later
qp = self.create_qp(self.ctx, qia, True, False, self.ib_port)
qp_attr, qp_init_attr = qp.query(e.IBV_QP_STATE | e.IBV_QP_CAP)
self.verify_qp_attrs(caps, e.IBV_QPS_RESET, qp_init_attr,
qp_attr)
def create_qp_common_test(self, qp_type, qp_state, is_ex, with_attr):
"""
Common function used by create QP tests.
"""
with PD(self.ctx) as pd:
with CQ(self.ctx, 100, None, None, 0) as cq:
if qp_type == e.IBV_QPT_RAW_PACKET:
if not (u.is_eth(self.ctx, self.ib_port) and u.is_root()):
raise unittest.SkipTest(
'To Create RAW QP must be done by root on Ethernet link layer'
)
if is_ex:
qia = get_qp_init_attr_ex(cq, pd, self.attr, self.attr_ex,
qp_type)
creator = self.ctx
else:
qia = u.get_qp_init_attr(cq, self.attr)
qia.qp_type = qp_type
creator = pd
qp = self.create_qp(creator, qia, is_ex, with_attr,
self.ib_port)
qp_type_str = pu.qp_type_to_str(qp_type)
qp_state_str = pu.qp_state_to_str(qp_state)
assert qp.qp_state == qp_state, f'{qp_type_str} QP should have been in {qp_state_str}'
def test_create_ah_roce(self):
"""
Verify that AH can't be created without GRH in RoCE
"""
done = 0
for ctx, attr, attr_ex in self.devices:
pd = PD(ctx)
for port_num in range(1, 1 + attr.phys_port_cnt):
port_attr = ctx.query_port(port_num)
if port_attr.state != e.IBV_PORT_ACTIVE and \
port_attr.state != e.IBV_PORT_INIT:
continue
if port_attr.link_layer != e.IBV_LINK_LAYER_ETHERNET:
raise unittest.SkipTest('RoCE tests are only supported on Ethernet link layer')
ah_attr = AHAttr(is_global=0, port_num=port_num)
try:
ah = AH(pd, attr=ah_attr)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create AH is not supported')
assert 'Failed to create AH' in str(ex)
done +=1
else:
raise PyverbsError('Created a non-global AH on RoCE')
if done == 0:
raise unittest.SkipTest('No port is up, can\'t create AH')
def query_qp_common_test(self, qp_type):
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
port_num = 1
if qp_type == e.IBV_QPT_RAW_PACKET:
eth_port = 0
for i in range(1, attr.phys_port_cnt + 1):
if u.is_eth(ctx, i) and u.is_root():
eth_port = i
port_num = eth_port
break
if eth_port == 0:
raise unittest.SkipTest(
'To Create RAW QP must be done by root on Ethernet link layer'
)
# Legacy QP
qia = u.get_qp_init_attr(cq, attr)
qia.qp_type = qp_type
caps = qia.cap
qp = self.create_qp(pd, qia, False, False, port_num)
qp_attr, qp_init_attr = qp.query(e.IBV_QP_STATE
| e.IBV_QP_CAP)
self.verify_qp_attrs(caps, e.IBV_QPS_RESET, qp_init_attr,
qp_attr)
# Extended QP
qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex, qp_type)
caps = qia.cap # Save them to verify values later
qp = self.create_qp(ctx, qia, True, False, port_num)
qp_attr, qp_init_attr = qp.query(e.IBV_QP_STATE
| e.IBV_QP_CAP)
self.verify_qp_attrs(caps, e.IBV_QPS_RESET, qp_init_attr,
qp_attr)
def test_dm_bad_access(self):
"""
Test multiple types of bad access to the Device Memory. Device memory
access requests a 4B alignment. The test tries to access the DM
with bad alignment or outside of the allocated memory.
"""
ctx, _, _ = self.devices[0]
dm_size = 100
with d.DM(ctx, d.AllocDmAttr(length=dm_size)) as dm:
dm_access = e.IBV_ACCESS_ZERO_BASED | e.IBV_ACCESS_LOCAL_WRITE
dmmr = DMMR(PD(ctx), dm_size, dm_access, dm, 0)
access_cases = [
(DM_INVALID_ALIGNMENT,
4), # Valid length with unaligned offset
(4,
DM_INVALID_ALIGNMENT), # Valid offset with unaligned length
(dm_size + 4, 4), # Offset out of allocated memory
(0, dm_size + 4)
] # Length out of allocated memory
for case in access_cases:
offset, length = case
with self.assertRaisesRegex(PyverbsRDMAError,
'Failed to copy from dm'):
dmmr.read(offset=offset, length=length)
with self.assertRaisesRegex(PyverbsRDMAError,
'Failed to copy to dm'):
dmmr.write(data='s' * length, offset=offset, length=length)
def test_create_ah_roce(self):
"""
Verify that AH can't be created without GRH in RoCE
"""
done = 0
for ctx, attr, attr_ex in self.devices:
pd = PD(ctx)
for port_num in range(1, 1 + attr.phys_port_cnt):
port_attr = ctx.query_port(port_num)
if port_attr.state != e.IBV_PORT_ACTIVE and \
port_attr.state != e.IBV_PORT_INIT:
continue
if port_attr.link_layer == e.IBV_LINK_LAYER_INFINIBAND:
raise unittest.SkipTest(
'Can\'t run RoCE tests on IB link layer')
ah_attr = AHAttr(is_global=0, port_num=port_num)
try:
ah = AH(pd, attr=ah_attr)
except PyverbsError as err:
assert 'Failed to create AH' in err.args[0]
done += 1
else:
raise PyverbsError('Created a non-global AH on RoCE')
if done == 0:
raise unittest.SkipTest('No port is up, can\'t create AH')
def create_qp_common_test(self, qp_type, qp_state, is_ex, with_attr):
"""
Common function used by create QP tests.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
port_num = 1
if qp_type == e.IBV_QPT_RAW_PACKET:
eth_port = 0
for i in range(1, attr.phys_port_cnt + 1):
if u.is_eth(ctx, i) and u.is_root():
eth_port = i
port_num = eth_port
break
if eth_port == 0:
raise unittest.SkipTest(
'To Create RAW QP must be done by root on Ethernet link layer'
)
if is_ex:
qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex,
qp_type)
creator = ctx
else:
qia = u.get_qp_init_attr(cq, attr)
qia.qp_type = qp_type
creator = pd
qp = self.create_qp(creator, qia, is_ex, with_attr,
port_num)
qp_type_str = pu.qp_type_to_str(qp_type)
qp_state_str = pu.qp_state_to_str(qp_state)
assert qp.qp_state == qp_state, f'{qp_type_str} QP should have been in {qp_state_str}'
def test_create_qp_ex_with_attr(self):
"""
Test QP creation via ibv_create_qp_ex with a QPAttr object proivded.
Checked QP types are Raw Packet and UD. Raw Packet is skipped for
non-root users / Infiniband link layer.
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with CQ(ctx, 100, None, None, 0) as cq:
for i in range(1, attr.phys_port_cnt + 1):
qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex,
e.IBV_QPT_UD)
try:
with QP(ctx, qia, QPAttr()) as qp:
assert qp.qp_state == e.IBV_QPS_RTS, 'UD QP should have been in RTS'
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create QP with extended attrs is not supported')
raise ex
if is_eth(ctx, i) and is_root():
qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex,
e.IBV_QPT_RAW_PACKET)
try:
with QP(ctx, qia, QPAttr()) as qp:
assert qp.qp_state == e.IBV_QPS_RTS, 'Raw Packet QP should have been in RTS'
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create QP with extended attrs is not supported')
raise ex
def test_dmabuf_read(self):
"""
Test reading from DmaBufMR's buffer
"""
check_dmabuf_support(self.gpu)
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
check_dmabuf_mr_support(pd, self.gpu)
for i in range(10):
mr_len = u.get_mr_length()
flags = u.get_dmabuf_access_flags(ctx)
for f in flags:
for mr_off in [0, mr_len // 2]:
with DmaBufMR(pd,
mr_len,
f,
offset=mr_off,
gpu=self.gpu,
gtt=self.gtt) as mr:
write_len = min(random.randint(1, MAX_IO_LEN),
mr_len)
write_str = 'a' * write_len
mr.write(write_str, write_len)
read_len = random.randint(1, write_len)
offset = random.randint(
0, write_len - read_len)
read_str = mr.read(read_len, offset).decode()
assert read_str in write_str
def test_dereg_mr(self):
""" Test ibv_dereg_mr() """
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
flags = u.get_access_flags(ctx)
for f in flags:
with MR(pd, u.get_mr_length(), f) as mr:
mr.close()
def test_reg_mw_type2(self):
"""
Test ibv_alloc_mw() for type 2 MW
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with MW(pd, e.IBV_MW_TYPE_2):
pass
def test_dereg_mr(self):
""" Test ibv_dereg_mr() """
lst = d.get_device_list()
for dev in lst:
with d.Context(name=dev.name.decode()) as ctx:
with PD(ctx) as pd:
with MR(pd, u.get_mr_length(), u.get_access_flags()) as mr:
mr.close()
def test_dereg_mw_type2(self):
"""
Test ibv_dealloc_mw() for type 2 MW
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
with MW(pd, e.IBV_MW_TYPE_2) as mw:
mw.close()
def test_destroy_pd_twice(self):
"""
Test bad flow cases in destruction of a PD object
"""
with PD(self.ctx) as pd:
# Pyverbs supports multiple destruction of objects, we are
# not expecting an exception here.
pd.close()
pd.close()
def test_dealloc_pd():
"""
Test ibv_dealloc_pd()
"""
lst = d.get_device_list()
for dev in lst:
with d.Context(name=dev.name.decode()) as ctx:
with PD(ctx) as pd:
pd.close()
def test_alloc_pd(self):
"""
Test ibv_alloc_pd()
"""
lst = d.get_device_list()
for dev in lst:
with d.Context(name=dev.name.decode()) as ctx:
with PD(ctx):
pass
