def test_dm_atomic_ops(self): """ Tests "increment" and "test_and_set" MEMIC atomic operations. The test does two increments to the same buffer data and verifies the values using test_and_set. Then verifies that the latter op sets the buffer as expected. """ with d.DM(self.ctx, d.AllocDmAttr(length=self.dm_size)) as dm: # Set DM buffer to 0 dm.copy_to_dm(0, bytes(self.dm_size), self.dm_size) try: inc_addr = Mlx5DmOpAddr(dm, MEMIC_ATOMIC_INCREMENT) test_and_set_addr = Mlx5DmOpAddr(dm, MEMIC_ATOMIC_TEST_AND_SET) except PyverbsRDMAError as ex: if ex.error_code in [errno.EOPNOTSUPP, errno.EPROTONOSUPPORT]: raise unittest.SkipTest( 'MEMIC atomic operations are not supported') raise ex inc_addr.write(b'\x01') inc_addr.write(b'\x01') # Now we should read 0x02 and the memory set to ffs val = int.from_bytes(test_and_set_addr.read(1), 'big') self.assertEqual(val, 2) # Verify that TEST_AND_SET set the memory to ffs val = int.from_bytes(test_and_set_addr.read(1), 'big') self.assertEqual(val, 255) inc_addr.unmap(self.dm_size) test_and_set_addr.unmap(self.dm_size)
def test_dm_bad_access(self): """ Test multiple types of bad access to the Device Memory. Device memory access requests a 4B alignment. The test tries to access the DM with bad alignment or outside of the allocated memory. """ ctx, _, _ = self.devices[0] dm_size = 100 with d.DM(ctx, d.AllocDmAttr(length=dm_size)) as dm: dm_access = e.IBV_ACCESS_ZERO_BASED | e.IBV_ACCESS_LOCAL_WRITE dmmr = DMMR(PD(ctx), dm_size, dm_access, dm, 0) access_cases = [ (DM_INVALID_ALIGNMENT, 4), # Valid length with unaligned offset (4, DM_INVALID_ALIGNMENT), # Valid offset with unaligned length (dm_size + 4, 4), # Offset out of allocated memory (0, dm_size + 4) ] # Length out of allocated memory for case in access_cases: offset, length = case with self.assertRaisesRegex(PyverbsRDMAError, 'Failed to copy from dm'): dmmr.read(offset=offset, length=length) with self.assertRaisesRegex(PyverbsRDMAError, 'Failed to copy to dm'): dmmr.write(data='s' * length, offset=offset, length=length)
def test_parallel_dm_atomic_ops(self): """ Runs multiple threads that do test_and_set operation, followed by multiple threads that do increments of +1, to the same DM buffer. Then verifies that the buffer data was incremented as expected. """ threads = [] num_threads = 10 self.skip_queue = Queue() with d.DM(self.ctx, d.AllocDmAttr(length=self.dm_size)) as self.dm: for _ in range(num_threads): threads.append(Thread(target=self._read_from_op_addr)) threads[-1].start() for thread in threads: thread.join() threads = [] for _ in range(num_threads): threads.append(Thread(target=self._write_to_op_addr)) threads[-1].start() for thread in threads: thread.join() if not self.skip_queue.empty(): raise self.skip_queue.get() val = int.from_bytes(self._read_from_op_addr(), 'big') self.assertEqual( val, num_threads - 1, f'Read value is ({val}) is different than expected ({num_threads-1})' )
def get_dm_attrs(dm_len): """ Initializes an AllocDmAttr member with the given length and random alignment. It currently sets comp_mask = 0 since other comp_mask values are not supported. :param dm_len: :return: An initialized AllocDmAttr object """ align = random.randint(MIN_DM_LOG_ALIGN, MAX_DM_LOG_ALIGN) return d.AllocDmAttr(dm_len, align, 0)
def test_dm_bad_registration(self): """ Test bad Device Memory registration when trying to register bigger DMMR than the allocated DM. """ dm_size = 100 with d.DM(self.ctx, d.AllocDmAttr(length=dm_size)) as dm: dm_access = e.IBV_ACCESS_ZERO_BASED | e.IBV_ACCESS_LOCAL_WRITE with self.assertRaisesRegex(PyverbsRDMAError, 'Failed to register a device MR'): DMMR(PD(self.ctx), dm_size + 4, dm_access, dm, 0)
def test_create_dm_mr(self): max_dm_size = self.attr_ex.max_dm_size dm_access = e.IBV_ACCESS_ZERO_BASED | e.IBV_ACCESS_LOCAL_WRITE for dm_size in [4, max_dm_size/4, max_dm_size/2]: dm_size = dm_size - (dm_size % u.DM_ALIGNMENT) for dmmr_factor_size in [0.1, 0.5, 1]: dmmr_size = dm_size * dmmr_factor_size dmmr_size = dmmr_size - (dmmr_size % u.DM_ALIGNMENT) with d.DM(self.ctx, d.AllocDmAttr(length=dm_size)) as dm: DMMR(PD(self.ctx), dmmr_size, dm_access, dm, 0)
def create_mr(self): try: self.dm = d.DM(self.ctx, d.AllocDmAttr(length=self.msg_size)) access = e.IBV_ACCESS_ZERO_BASED | e.IBV_ACCESS_LOCAL_WRITE if self.remote_access: access |= e.IBV_ACCESS_REMOTE_WRITE self.mr = DMMR(self.pd, self.msg_size, access, self.dm, 0) except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest(f'Reg DMMR with access={access} is not supported') raise ex
def alloc_dm(self, res_queue, size): """ Alloc device memory. Used by multiple processes that allocate DMs in parallel. :param res_queue: Result Queue to return the result to the parent process. :param size: The DM allocation size. :return: None """ try: d.DM(self.ctx, d.AllocDmAttr(length=size)) except PyverbsError as err: res_queue.put(err.error_code) res_queue.put(0)
def test_import_dm(self): """ Creates a DM and imports it from a different (duplicated) Context. Then writes some data to the original DM, reads it from the imported DM and verifies that the read data is as expected. """ with d.DM(self.ctx, d.AllocDmAttr(length=self.dm_size)) as dm: cmd_fd_dup = os.dup(self.ctx.cmd_fd) try: imported_ctx = Context(cmd_fd=cmd_fd_dup) imported_dm = DM(imported_ctx, handle=dm.handle) except PyverbsRDMAError as ex: if ex.error_code in [errno.EOPNOTSUPP, errno.EPROTONOSUPPORT]: raise unittest.SkipTest('Some object imports are not supported') raise ex original_data = b'\xab' * self.dm_size dm.copy_to_dm(0, original_data, self.dm_size) read_data = imported_dm.copy_from_dm(0, self.dm_size) self.assertEqual(original_data, read_data) imported_dm.unimport()
def get_dm_attrs(dm_len): align = random.randint(MIN_DM_LOG_ALIGN, MAX_DM_LOG_ALIGN) # Comp mask != 0 is not supported return d.AllocDmAttr(dm_len, align, 0)