def test_dm_atomic_ops(self):
"""
Tests "increment" and "test_and_set" MEMIC atomic operations.
The test does two increments to the same buffer data and verifies the
values using test_and_set.
Then verifies that the latter op sets the buffer as expected.
"""
with d.DM(self.ctx, d.AllocDmAttr(length=self.dm_size)) as dm:
# Set DM buffer to 0
dm.copy_to_dm(0, bytes(self.dm_size), self.dm_size)
try:
inc_addr = Mlx5DmOpAddr(dm, MEMIC_ATOMIC_INCREMENT)
test_and_set_addr = Mlx5DmOpAddr(dm, MEMIC_ATOMIC_TEST_AND_SET)
except PyverbsRDMAError as ex:
if ex.error_code in [errno.EOPNOTSUPP, errno.EPROTONOSUPPORT]:
raise unittest.SkipTest(
'MEMIC atomic operations are not supported')
raise ex
inc_addr.write(b'\x01')
inc_addr.write(b'\x01')
# Now we should read 0x02 and the memory set to ffs
val = int.from_bytes(test_and_set_addr.read(1), 'big')
self.assertEqual(val, 2)
# Verify that TEST_AND_SET set the memory to ffs
val = int.from_bytes(test_and_set_addr.read(1), 'big')
self.assertEqual(val, 255)
inc_addr.unmap(self.dm_size)
test_and_set_addr.unmap(self.dm_size)
def test_dm_bad_access(self):
"""
Test multiple types of bad access to the Device Memory. Device memory
access requests a 4B alignment. The test tries to access the DM
with bad alignment or outside of the allocated memory.
"""
ctx, _, _ = self.devices[0]
dm_size = 100
with d.DM(ctx, d.AllocDmAttr(length=dm_size)) as dm:
dm_access = e.IBV_ACCESS_ZERO_BASED | e.IBV_ACCESS_LOCAL_WRITE
dmmr = DMMR(PD(ctx), dm_size, dm_access, dm, 0)
access_cases = [
(DM_INVALID_ALIGNMENT,
4), # Valid length with unaligned offset
(4,
DM_INVALID_ALIGNMENT), # Valid offset with unaligned length
(dm_size + 4, 4), # Offset out of allocated memory
(0, dm_size + 4)
] # Length out of allocated memory
for case in access_cases:
offset, length = case
with self.assertRaisesRegex(PyverbsRDMAError,
'Failed to copy from dm'):
dmmr.read(offset=offset, length=length)
with self.assertRaisesRegex(PyverbsRDMAError,
'Failed to copy to dm'):
dmmr.write(data='s' * length, offset=offset, length=length)
def test_parallel_dm_atomic_ops(self):
"""
Runs multiple threads that do test_and_set operation, followed by
multiple threads that do increments of +1, to the same DM buffer.
Then verifies that the buffer data was incremented as expected.
"""
threads = []
num_threads = 10
self.skip_queue = Queue()
with d.DM(self.ctx, d.AllocDmAttr(length=self.dm_size)) as self.dm:
for _ in range(num_threads):
threads.append(Thread(target=self._read_from_op_addr))
threads[-1].start()
for thread in threads:
thread.join()
threads = []
for _ in range(num_threads):
threads.append(Thread(target=self._write_to_op_addr))
threads[-1].start()
for thread in threads:
thread.join()
if not self.skip_queue.empty():
raise self.skip_queue.get()
val = int.from_bytes(self._read_from_op_addr(), 'big')
self.assertEqual(
val, num_threads - 1,
f'Read value is ({val}) is different than expected ({num_threads-1})'
)
def get_dm_attrs(dm_len):
"""
Initializes an AllocDmAttr member with the given length and random
alignment. It currently sets comp_mask = 0 since other comp_mask values
are not supported.
:param dm_len:
:return: An initialized AllocDmAttr object
"""
align = random.randint(MIN_DM_LOG_ALIGN, MAX_DM_LOG_ALIGN)
return d.AllocDmAttr(dm_len, align, 0)
def test_dm_bad_registration(self):
"""
Test bad Device Memory registration when trying to register bigger DMMR
than the allocated DM.
"""
dm_size = 100
with d.DM(self.ctx, d.AllocDmAttr(length=dm_size)) as dm:
dm_access = e.IBV_ACCESS_ZERO_BASED | e.IBV_ACCESS_LOCAL_WRITE
with self.assertRaisesRegex(PyverbsRDMAError, 'Failed to register a device MR'):
DMMR(PD(self.ctx), dm_size + 4, dm_access, dm, 0)
def test_create_dm_mr(self):
max_dm_size = self.attr_ex.max_dm_size
dm_access = e.IBV_ACCESS_ZERO_BASED | e.IBV_ACCESS_LOCAL_WRITE
for dm_size in [4, max_dm_size/4, max_dm_size/2]:
dm_size = dm_size - (dm_size % u.DM_ALIGNMENT)
for dmmr_factor_size in [0.1, 0.5, 1]:
dmmr_size = dm_size * dmmr_factor_size
dmmr_size = dmmr_size - (dmmr_size % u.DM_ALIGNMENT)
with d.DM(self.ctx, d.AllocDmAttr(length=dm_size)) as dm:
DMMR(PD(self.ctx), dmmr_size, dm_access, dm, 0)
def create_mr(self):
try:
self.dm = d.DM(self.ctx, d.AllocDmAttr(length=self.msg_size))
access = e.IBV_ACCESS_ZERO_BASED | e.IBV_ACCESS_LOCAL_WRITE
if self.remote_access:
access |= e.IBV_ACCESS_REMOTE_WRITE
self.mr = DMMR(self.pd, self.msg_size, access, self.dm, 0)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest(f'Reg DMMR with access={access} is not supported')
raise ex
def alloc_dm(self, res_queue, size):
"""
Alloc device memory. Used by multiple processes that allocate DMs in
parallel.
:param res_queue: Result Queue to return the result to the parent
process.
:param size: The DM allocation size.
:return: None
"""
try:
d.DM(self.ctx, d.AllocDmAttr(length=size))
except PyverbsError as err:
res_queue.put(err.error_code)
res_queue.put(0)
def test_import_dm(self):
"""
Creates a DM and imports it from a different (duplicated) Context.
Then writes some data to the original DM, reads it from the imported DM
and verifies that the read data is as expected.
"""
with d.DM(self.ctx, d.AllocDmAttr(length=self.dm_size)) as dm:
cmd_fd_dup = os.dup(self.ctx.cmd_fd)
try:
imported_ctx = Context(cmd_fd=cmd_fd_dup)
imported_dm = DM(imported_ctx, handle=dm.handle)
except PyverbsRDMAError as ex:
if ex.error_code in [errno.EOPNOTSUPP, errno.EPROTONOSUPPORT]:
raise unittest.SkipTest('Some object imports are not supported')
raise ex
original_data = b'\xab' * self.dm_size
dm.copy_to_dm(0, original_data, self.dm_size)
read_data = imported_dm.copy_from_dm(0, self.dm_size)
self.assertEqual(original_data, read_data)
imported_dm.unimport()
def get_dm_attrs(dm_len):
align = random.randint(MIN_DM_LOG_ALIGN, MAX_DM_LOG_ALIGN)
# Comp mask != 0 is not supported
return d.AllocDmAttr(dm_len, align, 0)
