class ImportResources(base_class): def __init__(self, cmd_fd, pd_handle, mr_handle, mr_addr=None, **kwargs): self.cmd_fd = cmd_fd self.pd_handle = pd_handle self.mr_handle = mr_handle self.mr_addr = mr_addr super(ImportResources, self).__init__(**kwargs) def create_context(self): try: self.ctx = Context(cmd_fd=self.cmd_fd) except u.PyverbsRDMAError as ex: if ex.error_code in [errno.EOPNOTSUPP, errno.EPROTONOSUPPORT]: raise unittest.SkipTest( 'Importing a device is not supported') raise ex def create_pd(self): self.pd = PD(self.ctx, handle=self.pd_handle) def create_mr(self): self.mr = MR(self.pd, handle=self.mr_handle, address=self.mr_addr) def unimport_resources(self): self.mr.unimport() self.pd.unimport() self.pd.close()
def test_modify_qp(self): """ Queries a QP after calling modify(). Verifies that its properties are as expected. """ for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: with CQ(ctx, 100, None, None, 0) as cq: is_ex = random.choice([True, False]) if is_ex: qia = get_qp_init_attr_ex(cq, pd, [e.IBV_QPT_UD], attr, attr_ex) else: qia = get_qp_init_attr(cq, [e.IBV_QPT_UD], attr) qp = QP(ctx, qia) if is_ex \ else QP(pd, qia) qa = QPAttr() qa.qkey = 0x123 qp.to_init(qa) attr, iattr = qp.query(e.IBV_QP_QKEY) assert attr.qkey == qa.qkey qp.to_rtr(qa) qa.sq_psn = 0x45 qp.to_rts(qa) attr, iattr = qp.query(e.IBV_QP_SQ_PSN) assert attr.sq_psn == qa.sq_psn qa.qp_state = e.IBV_QPS_RESET qp.modify(qa, e.IBV_QP_STATE) assert qp.qp_state == e.IBV_QPS_RESET
def set_bad_flow(self, bad_flow): self.bad_flow = bad_flow if self.bad_flow: self.pd_bad = PD(self.ctx) self.mr_bad_flow = False if bad_flow == DCI_TEST_BAD_FLOW_WITH_RESET: self.stream_check = True
def test_create_qp_no_attr(self): """ Test QP creation via ibv_create_qp without a QPAttr object proivded. Checked QP types are Raw Packet and UD. Raw Packet is skipped for non-root users / Infiniband link layer. """ for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: with CQ(ctx, 100, None, None, 0) as cq: for i in range(1, attr.phys_port_cnt + 1): qia = get_qp_init_attr(cq, attr) qia.qp_type = e.IBV_QPT_UD with QP(pd, qia) as qp: assert qp.qp_state == e.IBV_QPS_RESET, 'UD QP should have been in RESET' if u.is_eth(ctx, i) and u.is_root(): qia.qp_type = e.IBV_QPT_RAW_PACKET try: with QP(pd, qia) as qp: assert qp.qp_state == e.IBV_QPS_RESET, 'Raw Packet QP should have been in RESET' except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest( "Create Raw Packet QP is not supported" ) raise ex
def test_create_qp_no_attr_connected(self): """ Test QP creation via ibv_create_qp without a QPAttr object proivded. Checked QP types are RC and UC. """ for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: with CQ(ctx, 100, None, None, 0) as cq: qia = get_qp_init_attr(cq, attr) qia.qp_type = e.IBV_QPT_RC try: with QP(pd, qia) as qp: assert qp.qp_state == e.IBV_QPS_RESET, 'RC QP should have been in RESET' except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest( 'Create QP with RC attrs is not supported') raise ex qia.qp_type = e.IBV_QPT_UC try: with QP(pd, qia) as qp: assert qp.qp_state == e.IBV_QPS_RESET, 'UC QP should have been in RESET' except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest( 'Create QP with UC attrs is not supported') raise ex
def _create_mr_devx(self, rdma_device, buf_size): """This method create mkey and memory buffer using devx (part of ofed driver). """ mlx5dv_attr = Mlx5DVContextAttr() self.ctx = Mlx5Context(mlx5dv_attr, name=rdma_device) self.my_pd = PD(self.ctx) self.dv_pd = Mlx5DvObj(dve.MLX5DV_OBJ_PD, pd=self.my_pd).dvpd self.umem = self._create_umem(self.ctx, buf_size) self.mkey_ctx = SwMkc(umr_en=0, lr=1, lw=1, access_mode_1_0=0x1, pd=self.dv_pd.pdn, start_addr=self.umem.umem_addr, len=buf_size) mkey_in = CreateMkeyIn(sw_mkc=self.mkey_ctx, mkey_umem_id=self.umem.umem_id, mkey_umem_valid=1) self.mkey_obj = Mlx5DevxObj(self.ctx, mkey_in, len(CreateMkeyOut())) mkey_out = CreateMkeyOut(self.mkey_obj.out_view) if mkey_out.status: raise PyverbsRDMAError( "Failed to create mkey with syndrome {0}".format( mkey_out.syndrome)) mkey_index = mkey_out.mkey_index self.lkey = mkey_index << 8
def test_query_qp(self): """ Queries a QP after creation. Verifies that its properties are as expected. """ for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: with CQ(ctx, 100, None, None, 0) as cq: for i in range(1, attr.phys_port_cnt + 1): qpts = get_qp_types(ctx, i) for qpt in qpts: # Extended QP qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex, qpt) caps = qia.cap # Save them to verify values later qp = QP(ctx, qia) qp_attr, qp_init_attr = qp.query(e.IBV_QP_CUR_STATE | e.IBV_QP_CAP) verify_qp_attrs(caps, e.IBV_QPS_RESET, qp_init_attr, qp_attr) # Legacy QP qia = get_qp_init_attr(cq, attr) qia.qp_type = qpt caps = qia.cap # Save them to verify values later qp = QP(pd, qia) qp_attr, qp_init_attr = qp.query(e.IBV_QP_CUR_STATE | e.IBV_QP_CAP) verify_qp_attrs(caps, e.IBV_QPS_RESET, qp_init_attr, qp_attr)
def test_query_qp(self): """ Queries a QP after creation. Verifies that its properties are as expected. """ for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: with CQ(ctx, 100, None, None, 0) as cq: for i in range(1, attr.phys_port_cnt + 1): qpts = get_qp_types(ctx, i) for qpt in qpts: # Extended QP qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex, qpt) caps = qia.cap # Save them to verify values later try: qp = QP(ctx, qia) except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest('Create QP with extended attrs is not supported') raise ex qp_attr, qp_init_attr = qp.query(e.IBV_QP_CUR_STATE | e.IBV_QP_CAP) verify_qp_attrs(caps, e.IBV_QPS_RESET, qp_init_attr, qp_attr) # Legacy QP qia = get_qp_init_attr(cq, attr) qia.qp_type = qpt caps = qia.cap # Save them to verify values later qp = QP(pd, qia) qp_attr, qp_init_attr = qp.query(e.IBV_QP_CUR_STATE | e.IBV_QP_CAP) verify_qp_attrs(caps, e.IBV_QPS_RESET, qp_init_attr, qp_attr)
def test_multiple_pd_creation(self): """ Test multiple creations and destructions of a PD object """ for i in range(random.randint(1, 200)): with PD(self.ctx) as pd: pd.close()
def test_dmabuf_reg_mr_bad_flags(self): """ Verify that illegal flags combination fails as expected """ check_dmabuf_support(self.gpu) for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: check_dmabuf_mr_support(pd, self.gpu) for i in range(5): flags = random.sample([ e.IBV_ACCESS_REMOTE_WRITE, e.IBV_ACCESS_REMOTE_ATOMIC ], random.randint(1, 2)) mr_flags = 0 for i in flags: mr_flags += i.value try: DmaBufMR(pd, u.get_mr_length(), mr_flags, gpu=self.gpu, gtt=self.gtt) except PyverbsRDMAError as err: assert 'Failed to register a dma-buf MR' in err.args[0] else: raise PyverbsRDMAError( 'Registered a dma-buf MR with illegal falgs')
def test_mlx5_dek_management(self): """ Test crypto login and DEK management APIs. The test checks also that invalid actions are not permited, e.g, create DEK not in login session. """ try: self.pd = PD(self.ctx) cred_bytes = struct.pack('!6Q', *self.crypto_details['credential']) key = struct.pack('!5Q', *self.crypto_details['wrapped_key']) self.dek_init_attr = \ Mlx5DEKInitAttr(self.pd, key=key, key_size=dve.MLX5DV_CRYPTO_KEY_SIZE_128, key_purpose=dve.MLX5DV_CRYPTO_KEY_PURPOSE_AES_XTS, opaque=DEK_OPAQUE) self.verify_create_dek_out_of_login_session() self.verify_login_state(dve.MLX5DV_CRYPTO_LOGIN_STATE_NO_LOGIN) # Login to crypto session self.login_attr = Mlx5CryptoLoginAttr(cred_bytes) Mlx5Context.crypto_login(self.ctx, self.login_attr) self.verify_login_state(dve.MLX5DV_CRYPTO_LOGIN_STATE_VALID) self.verify_login_twice() self.dek = Mlx5DEK(self.ctx, self.dek_init_attr) self.verify_dek_opaque() self.dek.close() # Logout from crypto session Mlx5Context.crypto_logout(self.ctx) self.verify_login_state(dve.MLX5DV_CRYPTO_LOGIN_STATE_NO_LOGIN) except PyverbsRDMAError as ex: print(ex) if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest('Create crypto elements is not supported') raise ex
def test_alloc_pd(self): """ Test ibv_alloc_pd() """ for ctx, attr, attr_ex in self.devices: with PD(ctx): pass
def test_create_qp_ex_with_attr_connected(self): """ Test QP creation via ibv_create_qp_ex with a QPAttr object proivded. Checked QP type are RC and UC. """ for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: with CQ(ctx, 100, None, None, 0) as cq: qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex, e.IBV_QPT_RC) try: with QP(ctx, qia, QPAttr()) as qp: assert qp.qp_state == e.IBV_QPS_INIT, 'RC QP should have been in INIT' except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest('Create QP with extended attrs is not supported') raise ex qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex, e.IBV_QPT_UC) try: with QP(ctx, qia, QPAttr()) as qp: assert qp.qp_state == e.IBV_QPS_INIT, 'UC QP should have been in INIT' except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest('Create QP with extended attrs is not supported') raise ex
def test_dealloc_pd(self): """ Test ibv_dealloc_pd() """ for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: pd.close()
def query_qp_common_test(self, qp_type): with PD(self.ctx) as pd: with CQ(self.ctx, 100, None, None, 0) as cq: if qp_type == e.IBV_QPT_RAW_PACKET: if not (u.is_eth(self.ctx, self.ib_port) and u.is_root()): raise unittest.SkipTest( 'To Create RAW QP must be done by root on Ethernet link layer' ) # Legacy QP qia = u.get_qp_init_attr(cq, self.attr) qia.qp_type = qp_type caps = qia.cap qp = self.create_qp(pd, qia, False, False, self.ib_port) qp_attr, qp_init_attr = qp.query(e.IBV_QP_STATE | e.IBV_QP_CAP) self.verify_qp_attrs(caps, e.IBV_QPS_RESET, qp_init_attr, qp_attr) # Extended QP qia = get_qp_init_attr_ex(cq, pd, self.attr, self.attr_ex, qp_type) caps = qia.cap # Save them to verify values later qp = self.create_qp(self.ctx, qia, True, False, self.ib_port) qp_attr, qp_init_attr = qp.query(e.IBV_QP_STATE | e.IBV_QP_CAP) self.verify_qp_attrs(caps, e.IBV_QPS_RESET, qp_init_attr, qp_attr)
def create_qp_common_test(self, qp_type, qp_state, is_ex, with_attr): """ Common function used by create QP tests. """ with PD(self.ctx) as pd: with CQ(self.ctx, 100, None, None, 0) as cq: if qp_type == e.IBV_QPT_RAW_PACKET: if not (u.is_eth(self.ctx, self.ib_port) and u.is_root()): raise unittest.SkipTest( 'To Create RAW QP must be done by root on Ethernet link layer' ) if is_ex: qia = get_qp_init_attr_ex(cq, pd, self.attr, self.attr_ex, qp_type) creator = self.ctx else: qia = u.get_qp_init_attr(cq, self.attr) qia.qp_type = qp_type creator = pd qp = self.create_qp(creator, qia, is_ex, with_attr, self.ib_port) qp_type_str = pu.qp_type_to_str(qp_type) qp_state_str = pu.qp_state_to_str(qp_state) assert qp.qp_state == qp_state, f'{qp_type_str} QP should have been in {qp_state_str}'
def test_create_ah_roce(self): """ Verify that AH can't be created without GRH in RoCE """ done = 0 for ctx, attr, attr_ex in self.devices: pd = PD(ctx) for port_num in range(1, 1 + attr.phys_port_cnt): port_attr = ctx.query_port(port_num) if port_attr.state != e.IBV_PORT_ACTIVE and \ port_attr.state != e.IBV_PORT_INIT: continue if port_attr.link_layer != e.IBV_LINK_LAYER_ETHERNET: raise unittest.SkipTest('RoCE tests are only supported on Ethernet link layer') ah_attr = AHAttr(is_global=0, port_num=port_num) try: ah = AH(pd, attr=ah_attr) except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest('Create AH is not supported') assert 'Failed to create AH' in str(ex) done +=1 else: raise PyverbsError('Created a non-global AH on RoCE') if done == 0: raise unittest.SkipTest('No port is up, can\'t create AH')
def query_qp_common_test(self, qp_type): for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: with CQ(ctx, 100, None, None, 0) as cq: port_num = 1 if qp_type == e.IBV_QPT_RAW_PACKET: eth_port = 0 for i in range(1, attr.phys_port_cnt + 1): if u.is_eth(ctx, i) and u.is_root(): eth_port = i port_num = eth_port break if eth_port == 0: raise unittest.SkipTest( 'To Create RAW QP must be done by root on Ethernet link layer' ) # Legacy QP qia = u.get_qp_init_attr(cq, attr) qia.qp_type = qp_type caps = qia.cap qp = self.create_qp(pd, qia, False, False, port_num) qp_attr, qp_init_attr = qp.query(e.IBV_QP_STATE | e.IBV_QP_CAP) self.verify_qp_attrs(caps, e.IBV_QPS_RESET, qp_init_attr, qp_attr) # Extended QP qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex, qp_type) caps = qia.cap # Save them to verify values later qp = self.create_qp(ctx, qia, True, False, port_num) qp_attr, qp_init_attr = qp.query(e.IBV_QP_STATE | e.IBV_QP_CAP) self.verify_qp_attrs(caps, e.IBV_QPS_RESET, qp_init_attr, qp_attr)
def test_dm_bad_access(self): """ Test multiple types of bad access to the Device Memory. Device memory access requests a 4B alignment. The test tries to access the DM with bad alignment or outside of the allocated memory. """ ctx, _, _ = self.devices[0] dm_size = 100 with d.DM(ctx, d.AllocDmAttr(length=dm_size)) as dm: dm_access = e.IBV_ACCESS_ZERO_BASED | e.IBV_ACCESS_LOCAL_WRITE dmmr = DMMR(PD(ctx), dm_size, dm_access, dm, 0) access_cases = [ (DM_INVALID_ALIGNMENT, 4), # Valid length with unaligned offset (4, DM_INVALID_ALIGNMENT), # Valid offset with unaligned length (dm_size + 4, 4), # Offset out of allocated memory (0, dm_size + 4) ] # Length out of allocated memory for case in access_cases: offset, length = case with self.assertRaisesRegex(PyverbsRDMAError, 'Failed to copy from dm'): dmmr.read(offset=offset, length=length) with self.assertRaisesRegex(PyverbsRDMAError, 'Failed to copy to dm'): dmmr.write(data='s' * length, offset=offset, length=length)
def test_create_ah_roce(self): """ Verify that AH can't be created without GRH in RoCE """ done = 0 for ctx, attr, attr_ex in self.devices: pd = PD(ctx) for port_num in range(1, 1 + attr.phys_port_cnt): port_attr = ctx.query_port(port_num) if port_attr.state != e.IBV_PORT_ACTIVE and \ port_attr.state != e.IBV_PORT_INIT: continue if port_attr.link_layer == e.IBV_LINK_LAYER_INFINIBAND: raise unittest.SkipTest( 'Can\'t run RoCE tests on IB link layer') ah_attr = AHAttr(is_global=0, port_num=port_num) try: ah = AH(pd, attr=ah_attr) except PyverbsError as err: assert 'Failed to create AH' in err.args[0] done += 1 else: raise PyverbsError('Created a non-global AH on RoCE') if done == 0: raise unittest.SkipTest('No port is up, can\'t create AH')
def create_qp_common_test(self, qp_type, qp_state, is_ex, with_attr): """ Common function used by create QP tests. """ for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: with CQ(ctx, 100, None, None, 0) as cq: port_num = 1 if qp_type == e.IBV_QPT_RAW_PACKET: eth_port = 0 for i in range(1, attr.phys_port_cnt + 1): if u.is_eth(ctx, i) and u.is_root(): eth_port = i port_num = eth_port break if eth_port == 0: raise unittest.SkipTest( 'To Create RAW QP must be done by root on Ethernet link layer' ) if is_ex: qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex, qp_type) creator = ctx else: qia = u.get_qp_init_attr(cq, attr) qia.qp_type = qp_type creator = pd qp = self.create_qp(creator, qia, is_ex, with_attr, port_num) qp_type_str = pu.qp_type_to_str(qp_type) qp_state_str = pu.qp_state_to_str(qp_state) assert qp.qp_state == qp_state, f'{qp_type_str} QP should have been in {qp_state_str}'
def test_create_qp_ex_with_attr(self): """ Test QP creation via ibv_create_qp_ex with a QPAttr object proivded. Checked QP types are Raw Packet and UD. Raw Packet is skipped for non-root users / Infiniband link layer. """ for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: with CQ(ctx, 100, None, None, 0) as cq: for i in range(1, attr.phys_port_cnt + 1): qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex, e.IBV_QPT_UD) try: with QP(ctx, qia, QPAttr()) as qp: assert qp.qp_state == e.IBV_QPS_RTS, 'UD QP should have been in RTS' except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest('Create QP with extended attrs is not supported') raise ex if is_eth(ctx, i) and is_root(): qia = get_qp_init_attr_ex(cq, pd, attr, attr_ex, e.IBV_QPT_RAW_PACKET) try: with QP(ctx, qia, QPAttr()) as qp: assert qp.qp_state == e.IBV_QPS_RTS, 'Raw Packet QP should have been in RTS' except PyverbsRDMAError as ex: if ex.error_code == errno.EOPNOTSUPP: raise unittest.SkipTest('Create QP with extended attrs is not supported') raise ex
def test_dmabuf_read(self): """ Test reading from DmaBufMR's buffer """ check_dmabuf_support(self.gpu) for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: check_dmabuf_mr_support(pd, self.gpu) for i in range(10): mr_len = u.get_mr_length() flags = u.get_dmabuf_access_flags(ctx) for f in flags: for mr_off in [0, mr_len // 2]: with DmaBufMR(pd, mr_len, f, offset=mr_off, gpu=self.gpu, gtt=self.gtt) as mr: write_len = min(random.randint(1, MAX_IO_LEN), mr_len) write_str = 'a' * write_len mr.write(write_str, write_len) read_len = random.randint(1, write_len) offset = random.randint( 0, write_len - read_len) read_str = mr.read(read_len, offset).decode() assert read_str in write_str
def test_dereg_mr(self): """ Test ibv_dereg_mr() """ for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: flags = u.get_access_flags(ctx) for f in flags: with MR(pd, u.get_mr_length(), f) as mr: mr.close()
def test_reg_mw_type2(self): """ Test ibv_alloc_mw() for type 2 MW """ for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: with MW(pd, e.IBV_MW_TYPE_2): pass
def test_dereg_mr(self): """ Test ibv_dereg_mr() """ lst = d.get_device_list() for dev in lst: with d.Context(name=dev.name.decode()) as ctx: with PD(ctx) as pd: with MR(pd, u.get_mr_length(), u.get_access_flags()) as mr: mr.close()
def test_dereg_mw_type2(self): """ Test ibv_dealloc_mw() for type 2 MW """ for ctx, attr, attr_ex in self.devices: with PD(ctx) as pd: with MW(pd, e.IBV_MW_TYPE_2) as mw: mw.close()
def test_destroy_pd_twice(self): """ Test bad flow cases in destruction of a PD object """ with PD(self.ctx) as pd: # Pyverbs supports multiple destruction of objects, we are # not expecting an exception here. pd.close() pd.close()
def test_dealloc_pd(): """ Test ibv_dealloc_pd() """ lst = d.get_device_list() for dev in lst: with d.Context(name=dev.name.decode()) as ctx: with PD(ctx) as pd: pd.close()
def test_alloc_pd(self): """ Test ibv_alloc_pd() """ lst = d.get_device_list() for dev in lst: with d.Context(name=dev.name.decode()) as ctx: with PD(ctx): pass