def test_create_dm_bad_flow(self): """ test ibv_alloc_dm() with an illegal size and comp mask """ dm_len = self.attr_ex.max_dm_size + 1 dm_attrs = u.get_dm_attrs(dm_len) try: d.DM(self.ctx, dm_attrs) except PyverbsRDMAError as e: assert 'Failed to allocate device memory of size' in \ e.args[0] assert 'Max available size' in e.args[0] else: raise PyverbsError( 'Created a DM with size larger than max reported') dm_attrs.comp_mask = random.randint(1, 100) try: d.DM(self.ctx, dm_attrs) except PyverbsRDMAError as e: assert 'Failed to allocate device memory of size' in \ e.args[0] else: raise PyverbsError( 'Created a DM with illegal comp mask {c}'. \ format(c=dm_attrs.comp_mask))
def test_create_dm_bad_flow(self): """ test ibv_alloc_dm() with an illegal size and comp mask """ lst = d.get_device_list() for dev in lst: with d.Context(name=dev.name.decode()) as ctx: attr = ctx.query_device_ex() if attr.max_dm_size == 0: return dm_len = attr.max_dm_size + 1 dm_attrs = u.get_dm_attrs(dm_len) try: dm = d.DM(ctx, dm_attrs) except PyverbsRDMAError as e: assert 'Failed to allocate device memory of size' in e.args[ 0] assert 'Max available size' in e.args[0] else: raise PyverbsError( 'Created a DM with size larger than max reported') dm_attrs.comp_mask = random.randint(1, 100) try: dm = d.DM(ctx, dm_attrs) except PyverbsRDMAError as e: assert 'Failed to allocate device memory of size' in e.args[ 0] else: raise PyverbsError('Created a DM with illegal comp mask {c}'.\ format(c=dm_attrs.comp_mask))
def test_dm_bad_access(self): """ Test multiple types of bad access to the Device Memory. Device memory access requests a 4B alignment. The test tries to access the DM with bad alignment or outside of the allocated memory. """ ctx, _, _ = self.devices[0] dm_size = 100 with d.DM(ctx, d.AllocDmAttr(length=dm_size)) as dm: dm_access = e.IBV_ACCESS_ZERO_BASED | e.IBV_ACCESS_LOCAL_WRITE dmmr = DMMR(PD(ctx), dm_size, dm_access, dm, 0) access_cases = [ (DM_INVALID_ALIGNMENT, 4), # Valid length with unaligned offset (4, DM_INVALID_ALIGNMENT), # Valid offset with unaligned length (dm_size + 4, 4), # Offset out of allocated memory (0, dm_size + 4) ] # Length out of allocated memory for case in access_cases: offset, length = case with self.assertRaisesRegex(PyverbsRDMAError, 'Failed to copy from dm'): dmmr.read(offset=offset, length=length) with self.assertRaisesRegex(PyverbsRDMAError, 'Failed to copy to dm'): dmmr.write(data='s' * length, offset=offset, length=length)
def test_dm_atomic_ops(self): """ Tests "increment" and "test_and_set" MEMIC atomic operations. The test does two increments to the same buffer data and verifies the values using test_and_set. Then verifies that the latter op sets the buffer as expected. """ with d.DM(self.ctx, d.AllocDmAttr(length=self.dm_size)) as dm: # Set DM buffer to 0 dm.copy_to_dm(0, bytes(self.dm_size), self.dm_size) try: inc_addr = Mlx5DmOpAddr(dm, MEMIC_ATOMIC_INCREMENT) test_and_set_addr = Mlx5DmOpAddr(dm, MEMIC_ATOMIC_TEST_AND_SET) except PyverbsRDMAError as ex: if ex.error_code in [errno.EOPNOTSUPP, errno.EPROTONOSUPPORT]: raise unittest.SkipTest( 'MEMIC atomic operations are not supported') raise ex inc_addr.write(b'\x01') inc_addr.write(b'\x01') # Now we should read 0x02 and the memory set to ffs val = int.from_bytes(test_and_set_addr.read(1), 'big') self.assertEqual(val, 2) # Verify that TEST_AND_SET set the memory to ffs val = int.from_bytes(test_and_set_addr.read(1), 'big') self.assertEqual(val, 255) inc_addr.unmap(self.dm_size) test_and_set_addr.unmap(self.dm_size)
def test_parallel_dm_atomic_ops(self): """ Runs multiple threads that do test_and_set operation, followed by multiple threads that do increments of +1, to the same DM buffer. Then verifies that the buffer data was incremented as expected. """ threads = [] num_threads = 10 self.skip_queue = Queue() with d.DM(self.ctx, d.AllocDmAttr(length=self.dm_size)) as self.dm: for _ in range(num_threads): threads.append(Thread(target=self._read_from_op_addr)) threads[-1].start() for thread in threads: thread.join() threads = [] for _ in range(num_threads): threads.append(Thread(target=self._write_to_op_addr)) threads[-1].start() for thread in threads: thread.join() if not self.skip_queue.empty(): raise self.skip_queue.get() val = int.from_bytes(self._read_from_op_addr(), 'big') self.assertEqual( val, num_threads - 1, f'Read value is ({val}) is different than expected ({num_threads-1})' )
def test_dm_write_bad_flow(self): """ Test writing to the device memory with bad offset and length """ lst = d.get_device_list() for dev in lst: with d.Context(name=dev.name.decode()) as ctx: attr = ctx.query_device_ex() if attr.max_dm_size == 0: return dm_len = random.randrange(u.MIN_DM_SIZE, attr.max_dm_size, u.DM_ALIGNMENT) dm_attrs = u.get_dm_attrs(dm_len) with d.DM(ctx, dm_attrs) as dm: data_length = random.randrange(4, dm_len, u.DM_ALIGNMENT) data_offset = random.randrange(0, dm_len - data_length, u.DM_ALIGNMENT) data_offset += 1 # offset needs to be a multiple of 4 data = u.get_data(data_length) try: dm.copy_to_dm(data_offset, data.encode(), data_length) except PyverbsRDMAError as e: assert 'Failed to copy to dm' in e.args[0] else: raise PyverbsError( 'Wrote to device memory with a bad offset')
def test_create_dm(self): """ test ibv_alloc_dm() """ dm_len = random.randrange(u.MIN_DM_SIZE, self.attr_ex.max_dm_size / 2, u.DM_ALIGNMENT) dm_attrs = u.get_dm_attrs(dm_len) with d.DM(self.ctx, dm_attrs): pass
def test_destroy_dm(self): """ test ibv_free_dm() """ dm_len = random.randrange(u.MIN_DM_SIZE, self.attr_ex.max_dm_size / 2, u.DM_ALIGNMENT) dm_attrs = u.get_dm_attrs(dm_len) dm = d.DM(self.ctx, dm_attrs) dm.close()
def test_destroy_dm_bad_flow(self): """ Test calling ibv_free_dm() twice """ dm_len = random.randrange(u.MIN_DM_SIZE, self.attr_ex.max_dm_size / 2, u.DM_ALIGNMENT) dm_attrs = u.get_dm_attrs(dm_len) dm = d.DM(self.ctx, dm_attrs) dm.close() dm.close()
def test_dm_bad_registration(self): """ Test bad Device Memory registration when trying to register bigger DMMR than the allocated DM. """ dm_size = 100 with d.DM(self.ctx, d.AllocDmAttr(length=dm_size)) as dm: dm_access = e.IBV_ACCESS_ZERO_BASED | e.IBV_ACCESS_LOCAL_WRITE with self.assertRaisesRegex(PyverbsRDMAError, 'Failed to register a device MR'): DMMR(PD(self.ctx), dm_size + 4, dm_access, dm, 0)
def test_create_dm_mr(self): max_dm_size = self.attr_ex.max_dm_size dm_access = e.IBV_ACCESS_ZERO_BASED | e.IBV_ACCESS_LOCAL_WRITE for dm_size in [4, max_dm_size/4, max_dm_size/2]: dm_size = dm_size - (dm_size % u.DM_ALIGNMENT) for dmmr_factor_size in [0.1, 0.5, 1]: dmmr_size = dm_size * dmmr_factor_size dmmr_size = dmmr_size - (dmmr_size % u.DM_ALIGNMENT) with d.DM(self.ctx, d.AllocDmAttr(length=dm_size)) as dm: DMMR(PD(self.ctx), dmmr_size, dm_access, dm, 0)
def create_mr(self): try: self.dm = d.DM(self.ctx, d.AllocDmAttr(length=self.msg_size)) access = e.IBV_ACCESS_ZERO_BASED | e.IBV_ACCESS_LOCAL_WRITE if self.remote_access: access |= e.IBV_ACCESS_REMOTE_WRITE self.mr = DMMR(self.pd, self.msg_size, access, self.dm, 0) except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest(f'Reg DMMR with access={access} is not supported') raise ex
def test_create_dm(self): """ test ibv_alloc_dm() """ for ctx, attr, attr_ex in self.devices: if attr_ex.max_dm_size == 0: return dm_len = random.randrange(u.MIN_DM_SIZE, attr_ex.max_dm_size, u.DM_ALIGNMENT) dm_attrs = u.get_dm_attrs(dm_len) with d.DM(ctx, dm_attrs): pass
def test_destroy_dm(self): """ test ibv_free_dm() """ for ctx, attr, attr_ex in self.devices: if attr_ex.max_dm_size == 0: raise unittest.SkipTest('Device memory is not supported') dm_len = random.randrange(u.MIN_DM_SIZE, attr_ex.max_dm_size / 2, u.DM_ALIGNMENT) dm_attrs = u.get_dm_attrs(dm_len) dm = d.DM(ctx, dm_attrs) dm.close()
def test_create_dm(self): """ test ibv_alloc_dm() """ for ctx, attr, attr_ex in self.devices: if attr_ex.max_dm_size == 0: raise unittest.SkipTest('Device memory is not supported') dm_len = random.randrange(u.MIN_DM_SIZE, attr_ex.max_dm_size / 2, u.DM_ALIGNMENT) dm_attrs = u.get_dm_attrs(dm_len) with d.DM(ctx, dm_attrs): pass
def test_destroy_dm(self): """ test ibv_free_dm() """ for ctx, attr, attr_ex in self.devices: if attr_ex.max_dm_size == 0: return dm_len = random.randrange(u.MIN_DM_SIZE, attr_ex.max_dm_size, u.DM_ALIGNMENT) dm_attrs = u.get_dm_attrs(dm_len) dm = d.DM(ctx, dm_attrs) dm.close()
def test_dm_write(self): """ Test writing to the device memory """ dm_len = random.randrange(u.MIN_DM_SIZE, self.attr_ex.max_dm_size / 2, u.DM_ALIGNMENT) dm_attrs = u.get_dm_attrs(dm_len) with d.DM(self.ctx, dm_attrs) as dm: data_length = random.randrange(4, dm_len, u.DM_ALIGNMENT) data_offset = random.randrange(0, dm_len - data_length, u.DM_ALIGNMENT) data = 'a' * data_length dm.copy_to_dm(data_offset, data.encode(), data_length)
def alloc_dm(self, res_queue, size): """ Alloc device memory. Used by multiple processes that allocate DMs in parallel. :param res_queue: Result Queue to return the result to the parent process. :param size: The DM allocation size. :return: None """ try: d.DM(self.ctx, d.AllocDmAttr(length=size)) except PyverbsError as err: res_queue.put(err.error_code) res_queue.put(0)
def test_dm_read(self): """ Test reading from the device memory """ dm_len = random.randrange(u.MIN_DM_SIZE, self.attr_ex.max_dm_size / 2, u.DM_ALIGNMENT) dm_attrs = u.get_dm_attrs(dm_len) with d.DM(self.ctx, dm_attrs) as dm: data_length = random.randrange(4, dm_len, u.DM_ALIGNMENT) data_offset = random.randrange(0, dm_len - data_length, u.DM_ALIGNMENT) data = 'a' * data_length dm.copy_to_dm(data_offset, data.encode(), data_length) read_str = dm.copy_from_dm(data_offset, data_length) assert read_str.decode() == data
def test_destroy_dm(self): """ test ibv_free_dm() """ lst = d.get_device_list() for dev in lst: with d.Context(name=dev.name.decode()) as ctx: attr = ctx.query_device_ex() if attr.max_dm_size == 0: return dm_len = random.randrange(u.MIN_DM_SIZE, attr.max_dm_size, u.DM_ALIGNMENT) dm_attrs = u.get_dm_attrs(dm_len) dm = d.DM(ctx, dm_attrs) dm.close()
def test_dm_write(self): """ Test writing to the device memory """ for ctx, attr, attr_ex in self.devices: if attr_ex.max_dm_size == 0: raise unittest.SkipTest('Device memory is not supported') dm_len = random.randrange(u.MIN_DM_SIZE, attr_ex.max_dm_size / 2, u.DM_ALIGNMENT) dm_attrs = u.get_dm_attrs(dm_len) with d.DM(ctx, dm_attrs) as dm: data_length = random.randrange(4, dm_len, u.DM_ALIGNMENT) data_offset = random.randrange(0, dm_len - data_length, u.DM_ALIGNMENT) data = 'a' * data_length dm.copy_to_dm(data_offset, data.encode(), data_length)
def test_create_dm_mr(self): """ Test ibv_reg_dm_mr """ for ctx, attr, attr_ex in self.devices: if attr_ex.max_dm_size == 0: return with PD(ctx) as pd: for i in range(10): dm_len = random.randrange(u.MIN_DM_SIZE, attr_ex.max_dm_size, u.DM_ALIGNMENT) dm_attrs = u.get_dm_attrs(dm_len) with d.DM(ctx, dm_attrs) as dm: dm_mr_len = random.randint(1, dm_len) dm_mr_offset = random.randint(0, (dm_len - dm_mr_len)) DMMR(pd, dm_mr_len, e.IBV_ACCESS_ZERO_BASED, dm=dm, offset=dm_mr_offset)
def test_dm_write_bad_flow(self): """ Test writing to the device memory with bad offset and length """ dm_len = random.randrange(u.MIN_DM_SIZE, self.attr_ex.max_dm_size / 2, u.DM_ALIGNMENT) dm_attrs = u.get_dm_attrs(dm_len) with d.DM(self.ctx, dm_attrs) as dm: data_length = random.randrange(4, dm_len, u.DM_ALIGNMENT) data_offset = random.randrange(0, dm_len - data_length, u.DM_ALIGNMENT) data_offset += 1 # offset needs to be a multiple of 4 data = 'a' * data_length try: dm.copy_to_dm(data_offset, data.encode(), data_length) except PyverbsRDMAError as e: assert 'Failed to copy to dm' in e.args[0] else: raise PyverbsError('Wrote to device memory with a bad offset')
def test_dm_write(self): """ Test writing to the device memory """ lst = d.get_device_list() for dev in lst: with d.Context(name=dev.name.decode()) as ctx: attr = ctx.query_device_ex() if attr.max_dm_size == 0: return dm_len = random.randrange(u.MIN_DM_SIZE, attr.max_dm_size, u.DM_ALIGNMENT) dm_attrs = u.get_dm_attrs(dm_len) with d.DM(ctx, dm_attrs) as dm: data_length = random.randrange(4, dm_len, u.DM_ALIGNMENT) data_offset = random.randrange(0, dm_len - data_length, u.DM_ALIGNMENT) data = u.get_data(data_length) dm.copy_to_dm(data_offset, data.encode(), data_length)
def test_import_dm(self): """ Creates a DM and imports it from a different (duplicated) Context. Then writes some data to the original DM, reads it from the imported DM and verifies that the read data is as expected. """ with d.DM(self.ctx, d.AllocDmAttr(length=self.dm_size)) as dm: cmd_fd_dup = os.dup(self.ctx.cmd_fd) try: imported_ctx = Context(cmd_fd=cmd_fd_dup) imported_dm = DM(imported_ctx, handle=dm.handle) except PyverbsRDMAError as ex: if ex.error_code in [errno.EOPNOTSUPP, errno.EPROTONOSUPPORT]: raise unittest.SkipTest('Some object imports are not supported') raise ex original_data = b'\xab' * self.dm_size dm.copy_to_dm(0, original_data, self.dm_size) read_data = imported_dm.copy_from_dm(0, self.dm_size) self.assertEqual(original_data, read_data) imported_dm.unimport()
def test_destroy_dm_mr(self): """ Test freeing of dm_mr """ for ctx, attr, attr_ex in self.devices: if attr_ex.max_dm_size == 0: raise unittest.SkipTest('Device memory is not supported') with PD(ctx) as pd: for i in range(10): dm_len = random.randrange(u.MIN_DM_SIZE, attr_ex.max_dm_size / 2, u.DM_ALIGNMENT) dm_attrs = u.get_dm_attrs(dm_len) with d.DM(ctx, dm_attrs) as dm: dm_mr_len = random.randint(1, dm_len) dm_mr_offset = random.randint(0, (dm_len - dm_mr_len)) dm_mr = DMMR(pd, dm_mr_len, e.IBV_ACCESS_ZERO_BASED, dm=dm, offset=dm_mr_offset) dm_mr.close()
def test_create_dm_mr(self): """ Test ibv_reg_dm_mr """ lst = d.get_device_list() for dev in lst: with d.Context(name=dev.name.decode()) as ctx: attr = ctx.query_device_ex() if attr.max_dm_size == 0: return with PD(ctx) as pd: dm_len = random.randrange(u.MIN_DM_SIZE, attr.max_dm_size, u.DM_ALIGNMENT) dm_attrs = u.get_dm_attrs(dm_len) with d.DM(ctx, dm_attrs) as dm: dm_mr_len = random.randint(1, dm_len) dm_mr_offset = random.randint(0, (dm_len - dm_mr_len)) dm_mr = DMMR(pd, dm_mr_len, e.IBV_ACCESS_ZERO_BASED, dm=dm, offset=dm_mr_offset)