class ImportResources(base_class):
def __init__(self,
cmd_fd,
pd_handle,
mr_handle,
mr_addr=None,
**kwargs):
self.cmd_fd = cmd_fd
self.pd_handle = pd_handle
self.mr_handle = mr_handle
self.mr_addr = mr_addr
super(ImportResources, self).__init__(**kwargs)
def create_context(self):
try:
self.ctx = Context(cmd_fd=self.cmd_fd)
except u.PyverbsRDMAError as ex:
if ex.error_code in [errno.EOPNOTSUPP, errno.EPROTONOSUPPORT]:
raise unittest.SkipTest(
'Importing a device is not supported')
raise ex
def create_pd(self):
self.pd = PD(self.ctx, handle=self.pd_handle)
def create_mr(self):
self.mr = MR(self.pd, handle=self.mr_handle, address=self.mr_addr)
def unimport_resources(self):
self.mr.unimport()
self.pd.unimport()
self.pd.close()
class MRRes(RCResources):
def __init__(self, dev_name, ib_port, gid_index,
mr_access=e.IBV_ACCESS_LOCAL_WRITE):
"""
Initialize MR resources based on RC resources that include RC QP.
:param dev_name: Device name to be used
:param ib_port: IB port of the device to use
:param gid_index: Which GID index to use
:param mr_access: The MR access
"""
self.mr_access = mr_access
super().__init__(dev_name=dev_name, ib_port=ib_port, gid_index=gid_index)
def create_mr(self):
try:
self.mr = MR(self.pd, self.msg_size, self.mr_access)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest(f'Reg MR with access ({self.mr_access}) is not supported')
raise ex
def create_qp_attr(self):
qp_attr = QPAttr(port_num=self.ib_port)
qp_access = e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_REMOTE_WRITE
qp_attr.qp_access_flags = qp_access
return qp_attr
def rereg_mr(self, flags, pd=None, addr=0, length=0, access=0):
try:
self.mr.rereg(flags, pd, addr, length, access)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest(f'Rereg MR is not supported ({str(ex)})')
raise ex
def create_mr(self):
try:
self.mr = MR(self.pd, self.msg_size, self.mr_access)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest(f'Reg MR with access ({self.mr_access}) is not supported')
raise ex
def test_reg_mr_bad_flags(self):
"""
Verify that illegal flags combination fails as expected
"""
with d.Context(name=self.dev_name) as ctx:
with PD(ctx) as pd:
with self.assertRaisesRegex(PyverbsRDMAError,
'Failed to register a MR'):
MR(pd, u.get_mr_length(), e.IBV_ACCESS_REMOTE_WRITE)
with self.assertRaisesRegex(PyverbsRDMAError,
'Failed to register a MR'):
MR(pd, u.get_mr_length(), e.IBV_ACCESS_REMOTE_ATOMIC)
def create_mr(self):
"""
Initializes self.mr with an MR of length <msg_size> - defined by each
test.
:return: None
"""
self.mr = MR(self.pd, self.msg_size, e.IBV_ACCESS_LOCAL_WRITE)
def test_dereg_mr(self):
""" Test ibv_dereg_mr() """
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
flags = u.get_access_flags(ctx)
for f in flags:
with MR(pd, u.get_mr_length(), f) as mr:
mr.close()
def test_dereg_mr(self):
""" Test ibv_dereg_mr() """
lst = d.get_device_list()
for dev in lst:
with d.Context(name=dev.name.decode()) as ctx:
with PD(ctx) as pd:
with MR(pd, u.get_mr_length(), u.get_access_flags()) as mr:
mr.close()
def create_mr(self):
access = e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_MW_BIND
try:
self.mr = MR(self.pd, self.msg_size, access)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Reg MR with MW access is not supported')
raise ex
def create_mr(self):
access = e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_ON_DEMAND
if self.is_huge:
access |= e.IBV_ACCESS_HUGETLB
self.mr = MR(self.pd,
self.msg_size,
access,
address=self.user_addr,
implicit=self.is_implicit)
def test_reg_mr_bad_flow(self):
""" Verify that trying to register a MR with None PD fails """
try:
mr = MR(None, random.randint(0, 10000), u.get_access_flags())
except TypeError as te:
assert 'expected pyverbs.pd.PD' in te.args[0]
assert 'got NoneType' in te.args[0]
else:
raise PyverbsRDMAErrno('Created a MR with None PD')
def create_mr(self):
try:
atomic_access = e.IBV_ACCESS_LOCAL_WRITE | \
e.IBV_ACCESS_REMOTE_ATOMIC
self.mr = MR(self.pd, self.msg_size, atomic_access)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest(f'Reg MR with access ({atomic_access}) is not supported')
raise ex
def test_reg_mr_bad_flow():
""" Verify that trying to register a MR with None PD fails """
try:
# Use the simplest access flags necessary
MR(None, random.randint(0, 10000), e.IBV_ACCESS_LOCAL_WRITE)
except TypeError as te:
assert 'expected pyverbs.pd.PD' in te.args[0]
assert 'got NoneType' in te.args[0]
else:
raise PyverbsRDMAErrno('Created a MR with None PD')
def test_buffer(self):
"""
Test reading buf property
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
length = u.get_mr_length()
flags = u.get_access_flags(ctx)
for f in flags:
with MR(pd, length, f) as mr:
mr.buf
def test_dereg_mr_twice(self):
""" Verify that explicit call to MR's close() doesn't fails """
lst = d.get_device_list()
for dev in lst:
with d.Context(name=dev.name.decode()) as ctx:
with PD(ctx) as pd:
with MR(pd, u.get_mr_length(), u.get_access_flags()) as mr:
# Pyverbs supports multiple destruction of objects, we are
# not expecting an exception here.
mr.close()
mr.close()
def test_dereg_mr_twice(self):
""" Verify that explicit call to MR's close() doesn't fails """
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
flags = u.get_access_flags(ctx)
for f in flags:
with MR(pd, u.get_mr_length(), f) as mr:
# Pyverbs supports multiple destruction of objects,
# we are not expecting an exception here.
mr.close()
mr.close()
def test_buffer(self):
"""
Test reading buf property
"""
lst = d.get_device_list()
for dev in lst:
with d.Context(name=dev.name.decode()) as ctx:
with PD(ctx) as pd:
length = u.get_mr_length()
with MR(pd, length, u.get_access_flags()) as mr:
buf = mr.buf
def test_write(self):
"""
Test writing to MR's buffer
"""
lst = d.get_device_list()
for dev in lst:
with d.Context(name=dev.name.decode()) as ctx:
with PD(ctx) as pd:
mr_len = u.get_mr_length()
with MR(pd, mr_len, u.get_access_flags()) as mr:
write_len = min(random.randint(1, MAX_IO_LEN), mr_len)
mr.write(u.get_data(write_len), write_len)
def create_custom_mr(agr_obj, additional_access_flags=0, size=None):
"""
Creates a memory region using the aggregation object's PD.
If size is None, the agr_obj's message size is used to set the MR's size.
The access flags are local write and the additional_access_flags.
:param agr_obj: The aggregation object that creates the MR
:param additional_access_flags: Addition access flags to set in the MR
:param size: MR's length. If None, agr_obj.msg_size is used.
"""
mr_length = size if size else agr_obj.msg_size
return MR(agr_obj.pd, mr_length,
e.IBV_ACCESS_LOCAL_WRITE | additional_access_flags)
def create_resources(self, bad_flow_type=0, **resource_arg):
"""
Creates DmaGga test resources that include a "server" resource that
can be used to send the MEMCPY WR, and a destination MR to copy data to.
The destination MR can be created on a different PD or with insufficient
access permissions, according to the bad_flow_type.
:param bad_flow_type: (Optional) An enum of BadFlowType that indicates
the bad flow type (default: 0 - good flow)
:param resource_arg: Dict of args that specify the resource specific
attributes.
:return: None
"""
self.server = Mlx5DmaResources(**self.dev_info, **resource_arg)
self.dest_pd = self.server.pd
dest_mr_access = e.IBV_ACCESS_LOCAL_WRITE
if bad_flow_type == BadFlowType.DIFFERENT_PD:
self.dest_pd = PD(self.server.ctx)
elif bad_flow_type == BadFlowType.MR_ILLEGAL_ACCESS:
dest_mr_access = e.IBV_ACCESS_REMOTE_READ
self.dest_mr = MR(self.dest_pd, self.server.msg_size, dest_mr_access)
# No need to connect the QPs
self.server.pre_run([0], [0])
def test_write(self):
"""
Test writing to MR's buffer
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
for i in range(10):
mr_len = u.get_mr_length()
flags = u.get_access_flags(ctx)
for f in flags:
with MR(pd, mr_len, f) as mr:
write_len = min(random.randint(1, MAX_IO_LEN),
mr_len)
mr.write('a' * write_len, write_len)
def test_read(self):
"""
Test reading from MR's buffer
"""
lst = d.get_device_list()
for dev in lst:
with d.Context(name=dev.name.decode()) as ctx:
with PD(ctx) as pd:
mr_len = u.get_mr_length()
with MR(pd, mr_len, u.get_access_flags()) as mr:
write_len = min(random.randint(1, MAX_IO_LEN), mr_len)
write_str = u.get_data(write_len)
mr.write(write_str, write_len)
read_len = random.randint(1, write_len)
offset = random.randint(0, write_len - read_len)
read_str = mr.read(read_len, offset).decode()
assert read_str in write_str
def create_custom_mr(agr_obj, additional_access_flags=0, size=None):
"""
Creates a memory region using the aggregation object's PD.
If size is None, the agr_obj's message size is used to set the MR's size.
The access flags are local write and the additional_access_flags.
:param agr_obj: The aggregation object that creates the MR
:param additional_access_flags: Addition access flags to set in the MR
:param size: MR's length. If None, agr_obj.msg_size is used.
"""
mr_length = size if size else agr_obj.msg_size
try:
return MR(agr_obj.pd, mr_length,
e.IBV_ACCESS_LOCAL_WRITE | additional_access_flags)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest(f'Create custom mr with additional access flags {additional_access_flags} is not supported')
raise ex
def test_reg_mr_bad_flags(self):
""" Verify that illegal flags combination fails as expected """
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
for i in range(5):
flags = random.sample([e.IBV_ACCESS_REMOTE_WRITE,
e.IBV_ACCESS_REMOTE_ATOMIC],
random.randint(1, 2))
mr_flags = 0
for i in flags:
mr_flags += i.value
try:
MR(pd, u.get_mr_length(), mr_flags)
except PyverbsRDMAError as err:
assert 'Failed to register a MR' in err.args[0]
else:
raise PyverbsRDMAError('Registered a MR with illegal falgs')
def test_reg_mr_bad_flags(self):
""" Verify that illegal flags combination fails as expected """
lst = d.get_device_list()
for dev in lst:
with d.Context(name=dev.name.decode()) as ctx:
with PD(ctx) as pd:
flags = random.sample([
e.IBV_ACCESS_REMOTE_WRITE, e.IBV_ACCESS_REMOTE_ATOMIC
], random.randint(1, 2))
mr_flags = 0
for i in flags:
mr_flags += i.value
try:
mr = MR(pd, u.get_mr_length(), mr_flags)
except PyverbsRDMAError as err:
assert 'Failed to register a MR' in err.args[0]
else:
raise PyverbsRDMAError(
'Registered a MR with illegal falgs')
def test_read(self):
"""
Test reading from MR's buffer
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
for i in range(10):
mr_len = u.get_mr_length()
flags = u.get_access_flags(ctx)
for f in flags:
with MR(pd, mr_len, f) as mr:
write_len = min(random.randint(1, MAX_IO_LEN),
mr_len)
write_str = 'a' * write_len
mr.write(write_str, write_len)
read_len = random.randint(1, write_len)
offset = random.randint(0, write_len-read_len)
read_str = mr.read(read_len, offset).decode()
assert read_str in write_str
class DmaGgaMemcpy(Mlx5RDMATestCase):
def create_resources(self, bad_flow_type=0, **resource_arg):
"""
Creates DmaGga test resources that include a "server" resource that
can be used to send the MEMCPY WR, and a destination MR to copy data to.
The destination MR can be created on a different PD or with insufficient
access permissions, according to the bad_flow_type.
:param bad_flow_type: (Optional) An enum of BadFlowType that indicates
the bad flow type (default: 0 - good flow)
:param resource_arg: Dict of args that specify the resource specific
attributes.
:return: None
"""
self.server = Mlx5DmaResources(**self.dev_info, **resource_arg)
self.dest_pd = self.server.pd
dest_mr_access = e.IBV_ACCESS_LOCAL_WRITE
if bad_flow_type == BadFlowType.DIFFERENT_PD:
self.dest_pd = PD(self.server.ctx)
elif bad_flow_type == BadFlowType.MR_ILLEGAL_ACCESS:
dest_mr_access = e.IBV_ACCESS_REMOTE_READ
self.dest_mr = MR(self.dest_pd, self.server.msg_size, dest_mr_access)
# No need to connect the QPs
self.server.pre_run([0], [0])
def dma_memcpy(self, msg_size=1024):
"""
Creates resources and posts a memcpy WR.
After posting the WR, the WC opcode and the data are verified.
:param msg_size: Size of the data to be copied (in Bytes)
:return: None
"""
self.create_resources(msg_size=msg_size)
self.dest_mr.write('0' * msg_size, msg_size)
self.server.mr.write('s' * msg_size, msg_size)
self.server.qp.wr_start()
self.server.qp.wr_flags = e.IBV_SEND_SIGNALED
self.server.qp.wr_memcpy(self.dest_mr.lkey, self.dest_mr.buf,
self.server.mr.lkey, self.server.mr.buf,
msg_size)
self.server.qp.wr_complete()
u.poll_cq_ex(self.server.cq)
wc_opcode = self.server.cq.read_opcode()
self.assertEqual(wc_opcode, dve.MLX5DV_WC_MEMCPY,
'WC opcode validation failed')
self.assertEqual(self.dest_mr.read(msg_size, 0),
self.server.mr.read(msg_size, 0))
def dma_memcpy_bad_protection_flow(self, bad_flow_type):
"""
Creates resources with bad protection and posts a memcpy WR.
The bad protection is either a destination MR created on a different PD
or a destination MR created with insufficient access permissions.
:param bad_flow_type: An enum of BadFlowType that indicates the bad flow type
:return: None
"""
self.create_resources(bad_flow_type)
self.server.qp.wr_start()
self.server.qp.wr_flags = e.IBV_SEND_SIGNALED
self.server.qp.wr_memcpy(self.dest_mr.lkey, self.dest_mr.buf,
self.server.mr.lkey, self.server.mr.buf,
self.server.msg_size)
self.server.qp.wr_complete()
with self.assertRaises(PyverbsRDMAError):
u.poll_cq_ex(self.server.cq)
self.assertEqual(self.server.cq.status, e.IBV_WC_LOC_PROT_ERR,
'Expected CQE with Local Protection Error')
def test_dma_memcpy_data(self):
self.dma_memcpy()
def test_dma_memcpy_different_pd_bad_flow(self):
self.dma_memcpy_bad_protection_flow(BadFlowType.DIFFERENT_PD)
def test_dma_memcpy_protection_bad_flow(self):
self.dma_memcpy_bad_protection_flow(BadFlowType.MR_ILLEGAL_ACCESS)
def test_dma_memcpy_large_data_bad_flow(self):
"""
Bad flow test, testing DMA memcpy with data larger than the maximum
allowed size, according to the HCA capabilities.
:return: None
"""
try:
ctx = Mlx5Context(Mlx5DVContextAttr(), name=self.dev_name)
except PyverbsUserError as ex:
raise unittest.SkipTest(f'Could not open mlx5 context ({ex})')
except PyverbsRDMAError:
raise unittest.SkipTest('Opening mlx5 context is not supported')
max_size = ctx.query_mlx5_device(
dve.MLX5DV_CONTEXT_MASK_WR_MEMCPY_LENGTH).max_wr_memcpy_length
max_size = max_size if max_size else 1024
with self.assertRaises(PyverbsRDMAError):
self.dma_memcpy(max_size + 1)
def create_mr(self):
access = e.IBV_ACCESS_REMOTE_WRITE | e.IBV_ACCESS_LOCAL_WRITE
self.mr = MR(self.pd, self.msg_size, access)
def create_mr(self):
self.mr = MR(self.pd, handle=self.mr_handle, address=self.mr_addr)
def create_mr(self):
self.mr = MR(self.pd, self.msg_size + self.GRH_SIZE,
e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_ON_DEMAND)
def create_mr(self):
self.mr = MR(self.pd, self.msg_size,
e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_RELAXED_ORDERING)