def test_create_dm_bad_flow(self):
"""
test ibv_alloc_dm() with an illegal size and comp mask
"""
dm_len = self.attr_ex.max_dm_size + 1
dm_attrs = u.get_dm_attrs(dm_len)
try:
d.DM(self.ctx, dm_attrs)
except PyverbsRDMAError as e:
assert 'Failed to allocate device memory of size' in \
e.args[0]
assert 'Max available size' in e.args[0]
else:
raise PyverbsError(
'Created a DM with size larger than max reported')
dm_attrs.comp_mask = random.randint(1, 100)
try:
d.DM(self.ctx, dm_attrs)
except PyverbsRDMAError as e:
assert 'Failed to allocate device memory of size' in \
e.args[0]
else:
raise PyverbsError(
'Created a DM with illegal comp mask {c}'. \
format(c=dm_attrs.comp_mask))
def test_create_dm_bad_flow(self):
"""
test ibv_alloc_dm() with an illegal size and comp mask
"""
lst = d.get_device_list()
for dev in lst:
with d.Context(name=dev.name.decode()) as ctx:
attr = ctx.query_device_ex()
if attr.max_dm_size == 0:
return
dm_len = attr.max_dm_size + 1
dm_attrs = u.get_dm_attrs(dm_len)
try:
dm = d.DM(ctx, dm_attrs)
except PyverbsRDMAError as e:
assert 'Failed to allocate device memory of size' in e.args[
0]
assert 'Max available size' in e.args[0]
else:
raise PyverbsError(
'Created a DM with size larger than max reported')
dm_attrs.comp_mask = random.randint(1, 100)
try:
dm = d.DM(ctx, dm_attrs)
except PyverbsRDMAError as e:
assert 'Failed to allocate device memory of size' in e.args[
0]
else:
raise PyverbsError('Created a DM with illegal comp mask {c}'.\
format(c=dm_attrs.comp_mask))
def test_dm_bad_access(self):
"""
Test multiple types of bad access to the Device Memory. Device memory
access requests a 4B alignment. The test tries to access the DM
with bad alignment or outside of the allocated memory.
"""
ctx, _, _ = self.devices[0]
dm_size = 100
with d.DM(ctx, d.AllocDmAttr(length=dm_size)) as dm:
dm_access = e.IBV_ACCESS_ZERO_BASED | e.IBV_ACCESS_LOCAL_WRITE
dmmr = DMMR(PD(ctx), dm_size, dm_access, dm, 0)
access_cases = [
(DM_INVALID_ALIGNMENT,
4), # Valid length with unaligned offset
(4,
DM_INVALID_ALIGNMENT), # Valid offset with unaligned length
(dm_size + 4, 4), # Offset out of allocated memory
(0, dm_size + 4)
] # Length out of allocated memory
for case in access_cases:
offset, length = case
with self.assertRaisesRegex(PyverbsRDMAError,
'Failed to copy from dm'):
dmmr.read(offset=offset, length=length)
with self.assertRaisesRegex(PyverbsRDMAError,
'Failed to copy to dm'):
dmmr.write(data='s' * length, offset=offset, length=length)
def test_dm_atomic_ops(self):
"""
Tests "increment" and "test_and_set" MEMIC atomic operations.
The test does two increments to the same buffer data and verifies the
values using test_and_set.
Then verifies that the latter op sets the buffer as expected.
"""
with d.DM(self.ctx, d.AllocDmAttr(length=self.dm_size)) as dm:
# Set DM buffer to 0
dm.copy_to_dm(0, bytes(self.dm_size), self.dm_size)
try:
inc_addr = Mlx5DmOpAddr(dm, MEMIC_ATOMIC_INCREMENT)
test_and_set_addr = Mlx5DmOpAddr(dm, MEMIC_ATOMIC_TEST_AND_SET)
except PyverbsRDMAError as ex:
if ex.error_code in [errno.EOPNOTSUPP, errno.EPROTONOSUPPORT]:
raise unittest.SkipTest(
'MEMIC atomic operations are not supported')
raise ex
inc_addr.write(b'\x01')
inc_addr.write(b'\x01')
# Now we should read 0x02 and the memory set to ffs
val = int.from_bytes(test_and_set_addr.read(1), 'big')
self.assertEqual(val, 2)
# Verify that TEST_AND_SET set the memory to ffs
val = int.from_bytes(test_and_set_addr.read(1), 'big')
self.assertEqual(val, 255)
inc_addr.unmap(self.dm_size)
test_and_set_addr.unmap(self.dm_size)
def test_parallel_dm_atomic_ops(self):
"""
Runs multiple threads that do test_and_set operation, followed by
multiple threads that do increments of +1, to the same DM buffer.
Then verifies that the buffer data was incremented as expected.
"""
threads = []
num_threads = 10
self.skip_queue = Queue()
with d.DM(self.ctx, d.AllocDmAttr(length=self.dm_size)) as self.dm:
for _ in range(num_threads):
threads.append(Thread(target=self._read_from_op_addr))
threads[-1].start()
for thread in threads:
thread.join()
threads = []
for _ in range(num_threads):
threads.append(Thread(target=self._write_to_op_addr))
threads[-1].start()
for thread in threads:
thread.join()
if not self.skip_queue.empty():
raise self.skip_queue.get()
val = int.from_bytes(self._read_from_op_addr(), 'big')
self.assertEqual(
val, num_threads - 1,
f'Read value is ({val}) is different than expected ({num_threads-1})'
)
def test_dm_write_bad_flow(self):
"""
Test writing to the device memory with bad offset and length
"""
lst = d.get_device_list()
for dev in lst:
with d.Context(name=dev.name.decode()) as ctx:
attr = ctx.query_device_ex()
if attr.max_dm_size == 0:
return
dm_len = random.randrange(u.MIN_DM_SIZE, attr.max_dm_size,
u.DM_ALIGNMENT)
dm_attrs = u.get_dm_attrs(dm_len)
with d.DM(ctx, dm_attrs) as dm:
data_length = random.randrange(4, dm_len, u.DM_ALIGNMENT)
data_offset = random.randrange(0, dm_len - data_length,
u.DM_ALIGNMENT)
data_offset += 1 # offset needs to be a multiple of 4
data = u.get_data(data_length)
try:
dm.copy_to_dm(data_offset, data.encode(), data_length)
except PyverbsRDMAError as e:
assert 'Failed to copy to dm' in e.args[0]
else:
raise PyverbsError(
'Wrote to device memory with a bad offset')
def test_create_dm(self):
"""
test ibv_alloc_dm()
"""
dm_len = random.randrange(u.MIN_DM_SIZE, self.attr_ex.max_dm_size / 2,
u.DM_ALIGNMENT)
dm_attrs = u.get_dm_attrs(dm_len)
with d.DM(self.ctx, dm_attrs):
pass
def test_destroy_dm(self):
"""
test ibv_free_dm()
"""
dm_len = random.randrange(u.MIN_DM_SIZE, self.attr_ex.max_dm_size / 2,
u.DM_ALIGNMENT)
dm_attrs = u.get_dm_attrs(dm_len)
dm = d.DM(self.ctx, dm_attrs)
dm.close()
def test_destroy_dm_bad_flow(self):
"""
Test calling ibv_free_dm() twice
"""
dm_len = random.randrange(u.MIN_DM_SIZE, self.attr_ex.max_dm_size / 2,
u.DM_ALIGNMENT)
dm_attrs = u.get_dm_attrs(dm_len)
dm = d.DM(self.ctx, dm_attrs)
dm.close()
dm.close()
def test_dm_bad_registration(self):
"""
Test bad Device Memory registration when trying to register bigger DMMR
than the allocated DM.
"""
dm_size = 100
with d.DM(self.ctx, d.AllocDmAttr(length=dm_size)) as dm:
dm_access = e.IBV_ACCESS_ZERO_BASED | e.IBV_ACCESS_LOCAL_WRITE
with self.assertRaisesRegex(PyverbsRDMAError, 'Failed to register a device MR'):
DMMR(PD(self.ctx), dm_size + 4, dm_access, dm, 0)
def test_create_dm_mr(self):
max_dm_size = self.attr_ex.max_dm_size
dm_access = e.IBV_ACCESS_ZERO_BASED | e.IBV_ACCESS_LOCAL_WRITE
for dm_size in [4, max_dm_size/4, max_dm_size/2]:
dm_size = dm_size - (dm_size % u.DM_ALIGNMENT)
for dmmr_factor_size in [0.1, 0.5, 1]:
dmmr_size = dm_size * dmmr_factor_size
dmmr_size = dmmr_size - (dmmr_size % u.DM_ALIGNMENT)
with d.DM(self.ctx, d.AllocDmAttr(length=dm_size)) as dm:
DMMR(PD(self.ctx), dmmr_size, dm_access, dm, 0)
def create_mr(self):
try:
self.dm = d.DM(self.ctx, d.AllocDmAttr(length=self.msg_size))
access = e.IBV_ACCESS_ZERO_BASED | e.IBV_ACCESS_LOCAL_WRITE
if self.remote_access:
access |= e.IBV_ACCESS_REMOTE_WRITE
self.mr = DMMR(self.pd, self.msg_size, access, self.dm, 0)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest(f'Reg DMMR with access={access} is not supported')
raise ex
def test_create_dm(self):
"""
test ibv_alloc_dm()
"""
for ctx, attr, attr_ex in self.devices:
if attr_ex.max_dm_size == 0:
return
dm_len = random.randrange(u.MIN_DM_SIZE, attr_ex.max_dm_size,
u.DM_ALIGNMENT)
dm_attrs = u.get_dm_attrs(dm_len)
with d.DM(ctx, dm_attrs):
pass
def test_destroy_dm(self):
"""
test ibv_free_dm()
"""
for ctx, attr, attr_ex in self.devices:
if attr_ex.max_dm_size == 0:
raise unittest.SkipTest('Device memory is not supported')
dm_len = random.randrange(u.MIN_DM_SIZE, attr_ex.max_dm_size / 2,
u.DM_ALIGNMENT)
dm_attrs = u.get_dm_attrs(dm_len)
dm = d.DM(ctx, dm_attrs)
dm.close()
def test_create_dm(self):
"""
test ibv_alloc_dm()
"""
for ctx, attr, attr_ex in self.devices:
if attr_ex.max_dm_size == 0:
raise unittest.SkipTest('Device memory is not supported')
dm_len = random.randrange(u.MIN_DM_SIZE, attr_ex.max_dm_size / 2,
u.DM_ALIGNMENT)
dm_attrs = u.get_dm_attrs(dm_len)
with d.DM(ctx, dm_attrs):
pass
def test_destroy_dm(self):
"""
test ibv_free_dm()
"""
for ctx, attr, attr_ex in self.devices:
if attr_ex.max_dm_size == 0:
return
dm_len = random.randrange(u.MIN_DM_SIZE, attr_ex.max_dm_size,
u.DM_ALIGNMENT)
dm_attrs = u.get_dm_attrs(dm_len)
dm = d.DM(ctx, dm_attrs)
dm.close()
def test_dm_write(self):
"""
Test writing to the device memory
"""
dm_len = random.randrange(u.MIN_DM_SIZE, self.attr_ex.max_dm_size / 2,
u.DM_ALIGNMENT)
dm_attrs = u.get_dm_attrs(dm_len)
with d.DM(self.ctx, dm_attrs) as dm:
data_length = random.randrange(4, dm_len, u.DM_ALIGNMENT)
data_offset = random.randrange(0, dm_len - data_length,
u.DM_ALIGNMENT)
data = 'a' * data_length
dm.copy_to_dm(data_offset, data.encode(), data_length)
def alloc_dm(self, res_queue, size):
"""
Alloc device memory. Used by multiple processes that allocate DMs in
parallel.
:param res_queue: Result Queue to return the result to the parent
process.
:param size: The DM allocation size.
:return: None
"""
try:
d.DM(self.ctx, d.AllocDmAttr(length=size))
except PyverbsError as err:
res_queue.put(err.error_code)
res_queue.put(0)
def test_dm_read(self):
"""
Test reading from the device memory
"""
dm_len = random.randrange(u.MIN_DM_SIZE, self.attr_ex.max_dm_size / 2,
u.DM_ALIGNMENT)
dm_attrs = u.get_dm_attrs(dm_len)
with d.DM(self.ctx, dm_attrs) as dm:
data_length = random.randrange(4, dm_len, u.DM_ALIGNMENT)
data_offset = random.randrange(0, dm_len - data_length,
u.DM_ALIGNMENT)
data = 'a' * data_length
dm.copy_to_dm(data_offset, data.encode(), data_length)
read_str = dm.copy_from_dm(data_offset, data_length)
assert read_str.decode() == data
def test_destroy_dm(self):
"""
test ibv_free_dm()
"""
lst = d.get_device_list()
for dev in lst:
with d.Context(name=dev.name.decode()) as ctx:
attr = ctx.query_device_ex()
if attr.max_dm_size == 0:
return
dm_len = random.randrange(u.MIN_DM_SIZE, attr.max_dm_size,
u.DM_ALIGNMENT)
dm_attrs = u.get_dm_attrs(dm_len)
dm = d.DM(ctx, dm_attrs)
dm.close()
def test_dm_write(self):
"""
Test writing to the device memory
"""
for ctx, attr, attr_ex in self.devices:
if attr_ex.max_dm_size == 0:
raise unittest.SkipTest('Device memory is not supported')
dm_len = random.randrange(u.MIN_DM_SIZE, attr_ex.max_dm_size / 2,
u.DM_ALIGNMENT)
dm_attrs = u.get_dm_attrs(dm_len)
with d.DM(ctx, dm_attrs) as dm:
data_length = random.randrange(4, dm_len, u.DM_ALIGNMENT)
data_offset = random.randrange(0, dm_len - data_length,
u.DM_ALIGNMENT)
data = 'a' * data_length
dm.copy_to_dm(data_offset, data.encode(), data_length)
def test_create_dm_mr(self):
"""
Test ibv_reg_dm_mr
"""
for ctx, attr, attr_ex in self.devices:
if attr_ex.max_dm_size == 0:
return
with PD(ctx) as pd:
for i in range(10):
dm_len = random.randrange(u.MIN_DM_SIZE, attr_ex.max_dm_size,
u.DM_ALIGNMENT)
dm_attrs = u.get_dm_attrs(dm_len)
with d.DM(ctx, dm_attrs) as dm:
dm_mr_len = random.randint(1, dm_len)
dm_mr_offset = random.randint(0, (dm_len - dm_mr_len))
DMMR(pd, dm_mr_len, e.IBV_ACCESS_ZERO_BASED, dm=dm,
offset=dm_mr_offset)
def test_dm_write_bad_flow(self):
"""
Test writing to the device memory with bad offset and length
"""
dm_len = random.randrange(u.MIN_DM_SIZE, self.attr_ex.max_dm_size / 2,
u.DM_ALIGNMENT)
dm_attrs = u.get_dm_attrs(dm_len)
with d.DM(self.ctx, dm_attrs) as dm:
data_length = random.randrange(4, dm_len, u.DM_ALIGNMENT)
data_offset = random.randrange(0, dm_len - data_length,
u.DM_ALIGNMENT)
data_offset += 1 # offset needs to be a multiple of 4
data = 'a' * data_length
try:
dm.copy_to_dm(data_offset, data.encode(), data_length)
except PyverbsRDMAError as e:
assert 'Failed to copy to dm' in e.args[0]
else:
raise PyverbsError('Wrote to device memory with a bad offset')
def test_dm_write(self):
"""
Test writing to the device memory
"""
lst = d.get_device_list()
for dev in lst:
with d.Context(name=dev.name.decode()) as ctx:
attr = ctx.query_device_ex()
if attr.max_dm_size == 0:
return
dm_len = random.randrange(u.MIN_DM_SIZE, attr.max_dm_size,
u.DM_ALIGNMENT)
dm_attrs = u.get_dm_attrs(dm_len)
with d.DM(ctx, dm_attrs) as dm:
data_length = random.randrange(4, dm_len, u.DM_ALIGNMENT)
data_offset = random.randrange(0, dm_len - data_length,
u.DM_ALIGNMENT)
data = u.get_data(data_length)
dm.copy_to_dm(data_offset, data.encode(), data_length)
def test_import_dm(self):
"""
Creates a DM and imports it from a different (duplicated) Context.
Then writes some data to the original DM, reads it from the imported DM
and verifies that the read data is as expected.
"""
with d.DM(self.ctx, d.AllocDmAttr(length=self.dm_size)) as dm:
cmd_fd_dup = os.dup(self.ctx.cmd_fd)
try:
imported_ctx = Context(cmd_fd=cmd_fd_dup)
imported_dm = DM(imported_ctx, handle=dm.handle)
except PyverbsRDMAError as ex:
if ex.error_code in [errno.EOPNOTSUPP, errno.EPROTONOSUPPORT]:
raise unittest.SkipTest('Some object imports are not supported')
raise ex
original_data = b'\xab' * self.dm_size
dm.copy_to_dm(0, original_data, self.dm_size)
read_data = imported_dm.copy_from_dm(0, self.dm_size)
self.assertEqual(original_data, read_data)
imported_dm.unimport()
def test_destroy_dm_mr(self):
"""
Test freeing of dm_mr
"""
for ctx, attr, attr_ex in self.devices:
if attr_ex.max_dm_size == 0:
raise unittest.SkipTest('Device memory is not supported')
with PD(ctx) as pd:
for i in range(10):
dm_len = random.randrange(u.MIN_DM_SIZE,
attr_ex.max_dm_size / 2,
u.DM_ALIGNMENT)
dm_attrs = u.get_dm_attrs(dm_len)
with d.DM(ctx, dm_attrs) as dm:
dm_mr_len = random.randint(1, dm_len)
dm_mr_offset = random.randint(0, (dm_len - dm_mr_len))
dm_mr = DMMR(pd,
dm_mr_len,
e.IBV_ACCESS_ZERO_BASED,
dm=dm,
offset=dm_mr_offset)
dm_mr.close()
def test_create_dm_mr(self):
"""
Test ibv_reg_dm_mr
"""
lst = d.get_device_list()
for dev in lst:
with d.Context(name=dev.name.decode()) as ctx:
attr = ctx.query_device_ex()
if attr.max_dm_size == 0:
return
with PD(ctx) as pd:
dm_len = random.randrange(u.MIN_DM_SIZE, attr.max_dm_size,
u.DM_ALIGNMENT)
dm_attrs = u.get_dm_attrs(dm_len)
with d.DM(ctx, dm_attrs) as dm:
dm_mr_len = random.randint(1, dm_len)
dm_mr_offset = random.randint(0, (dm_len - dm_mr_len))
dm_mr = DMMR(pd,
dm_mr_len,
e.IBV_ACCESS_ZERO_BASED,
dm=dm,
offset=dm_mr_offset)
