def _cmid_server_traffic(self, multicast=False):
"""
RDMACM server side traffic function which sends and receives a message,
and then validates the received message. This traffic method uses the
RDMACM API for send, recv and get_completion.
:return: None
"""
grh_offset = GRH_SIZE if self.cm_res.qp_type == e.IBV_QPT_UD else 0
send_msg = (self.cm_res.msg_size + grh_offset) * 's'
cmid = self.cm_res.child_id if not multicast else self.cm_res.cmid
for _ in range(self.cm_res.num_msgs):
cmid.post_recv(self.cm_res.mr)
self.syncer.wait()
self.syncer.wait()
wc = cmid.get_recv_comp()
msg_received = self.cm_res.mr.read(self.cm_res.msg_size,
grh_offset)
validate(msg_received, True, self.cm_res.msg_size)
if self.cm_res.port_space == ce.RDMA_PS_TCP:
self.cm_res.mr.write(send_msg, self.cm_res.msg_size)
cmid.post_send(self.cm_res.mr)
else:
if multicast:
ah = AH(cmid.pd, attr=self.cm_res.ud_params.ah_attr)
rqpn = MULTICAST_QPN
else:
ah = AH(cmid.pd, wc=wc, port_num=1, grh=self.cm_res.mr.buf)
rqpn = self.cm_res.remote_qpn
self.cm_res.mr.write(send_msg, self.cm_res.msg_size + GRH_SIZE)
cmid.post_ud_send(self.cm_res.mr,
ah,
rqpn=rqpn,
length=self.cm_res.msg_size)
cmid.get_send_comp()
self.syncer.wait()
def test_create_ah_roce(self):
"""
Verify that AH can't be created without GRH in RoCE
"""
done = 0
for ctx, attr, attr_ex in self.devices:
pd = PD(ctx)
for port_num in range(1, 1 + attr.phys_port_cnt):
port_attr = ctx.query_port(port_num)
if port_attr.state != e.IBV_PORT_ACTIVE and \
port_attr.state != e.IBV_PORT_INIT:
continue
if port_attr.link_layer != e.IBV_LINK_LAYER_ETHERNET:
raise unittest.SkipTest('RoCE tests are only supported on Ethernet link layer')
ah_attr = AHAttr(is_global=0, port_num=port_num)
try:
ah = AH(pd, attr=ah_attr)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create AH is not supported')
assert 'Failed to create AH' in str(ex)
done +=1
else:
raise PyverbsError('Created a non-global AH on RoCE')
if done == 0:
raise unittest.SkipTest('No port is up, can\'t create AH')
def test_create_ah_roce(self):
"""
Verify that AH can't be created without GRH in RoCE
"""
done = 0
for ctx, attr, attr_ex in self.devices:
pd = PD(ctx)
for port_num in range(1, 1 + attr.phys_port_cnt):
port_attr = ctx.query_port(port_num)
if port_attr.state != e.IBV_PORT_ACTIVE and \
port_attr.state != e.IBV_PORT_INIT:
continue
if port_attr.link_layer == e.IBV_LINK_LAYER_INFINIBAND:
raise unittest.SkipTest(
'Can\'t run RoCE tests on IB link layer')
ah_attr = AHAttr(is_global=0, port_num=port_num)
try:
ah = AH(pd, attr=ah_attr)
except PyverbsError as err:
assert 'Failed to create AH' in err.args[0]
done += 1
else:
raise PyverbsError('Created a non-global AH on RoCE')
if done == 0:
raise unittest.SkipTest('No port is up, can\'t create AH')
def get_global_ah(agr_obj, gid_index, port):
gr = GlobalRoute(dgid=agr_obj.ctx.query_gid(port, gid_index),
sgid_index=gid_index)
ah_attr = AHAttr(port_num=port,
is_global=1,
gr=gr,
dlid=agr_obj.port_attr.lid)
return AH(agr_obj.pd, attr=ah_attr)
def test_destroy_ah(self):
"""
Test ibv_destroy_ah.
"""
for ctx, attr, attr_ex in self.devices:
pd = PD(ctx)
for port_num in range(1, 1 + attr.phys_port_cnt):
gr = get_global_route(ctx)
ah_attr = AHAttr(gr=gr, is_global=1, port_num=port_num)
with AH(pd, attr=ah_attr) as ah:
ah.close()
def _ext_qp_client_traffic(self):
self.cm_res.remote_dct_num = self.cm_res.remote_qpn
_, send_wr = u.get_send_elements(self.cm_res, self.cm_res.passive)
ah = AH(self.cm_res.cmid.pd, attr=self.cm_res.remote_ah)
self.syncer.wait()
for send_idx in range(self.cm_res.num_msgs):
dci_idx = send_idx % len(self.cm_res.qps)
u.post_send_ex(self.cm_res,
send_wr,
e.IBV_QP_EX_WITH_SEND,
ah=ah,
qp_idx=dci_idx)
u.poll_cq(self.cm_res.cq)
def test_create_ah(self):
"""
Test ibv_create_ah.
"""
self.verify_state(self.ctx)
gr = u.get_global_route(self.ctx, port_num=self.ib_port)
ah_attr = AHAttr(gr=gr, is_global=1, port_num=self.ib_port)
pd = PD(self.ctx)
try:
AH(pd, attr=ah_attr)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create AH is not supported')
raise ex
def test_create_ah_roce(self):
"""
Verify that AH can't be created without GRH in RoCE
"""
self.verify_link_layer_ether(self.ctx)
self.verify_state(self.ctx)
pd = PD(self.ctx)
ah_attr = AHAttr(is_global=0, port_num=self.ib_port)
try:
AH(pd, attr=ah_attr)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create AH is not supported')
assert 'Failed to create AH' in str(ex)
else:
raise PyverbsError(f'Successfully created a non-global AH on RoCE port={self.ib_port}')
def test_create_ah(self):
"""
Test ibv_create_ah.
"""
self.verify_state(self.ctx)
gr = u.get_global_route(self.ctx, port_num=self.ib_port)
port_attrs = self.ctx.query_port(self.ib_port)
dlid = port_attrs.lid if port_attrs.link_layer == e.IBV_LINK_LAYER_INFINIBAND else 0
ah_attr = AHAttr(dlid=dlid, gr=gr, is_global=1, port_num=self.ib_port)
pd = PD(self.ctx)
try:
AH(pd, attr=ah_attr)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create AH is not supported')
raise ex
def test_destroy_ah(self):
"""
Test ibv_destroy_ah.
"""
for ctx, _, _ in self.devices:
self.verify_state(ctx)
gr = u.get_global_route(ctx, port_num=self.ib_port)
ah_attr = AHAttr(gr=gr, is_global=1, port_num=self.ib_port)
pd = PD(ctx)
try:
with AH(pd, attr=ah_attr) as ah:
ah.close()
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create AH is not supported')
raise ex
def test_create_ah(self):
"""
Test ibv_create_ah.
"""
done = 0
for ctx, attr, attr_ex in self.devices:
pd = PD(ctx)
for port_num in range(1, 1 + attr.phys_port_cnt):
state = ctx.query_port(port_num).state
if state != e.IBV_PORT_ACTIVE and state != e.IBV_PORT_INIT:
continue
gr = get_global_route(ctx, port_num=port_num)
ah_attr = AHAttr(gr=gr, is_global=1, port_num=port_num)
with AH(pd, attr=ah_attr):
done += 1
if done == 0:
raise unittest.SkipTest('No port is up, can\'t create AH')
def test_create_ah_roce(self):
"""
Verify that AH can't be created without GRH in RoCE
"""
for ctx, attr, attr_ex in self.devices:
pd = PD(ctx)
for port_num in range(1, 1 + attr.phys_port_cnt):
port_attr = ctx.query_port(port_num)
if port_attr.link_layer == e.IBV_LINK_LAYER_INFINIBAND:
return
ah_attr = AHAttr(is_global=0, port_num=port_num)
try:
ah = AH(pd, attr=ah_attr)
except PyverbsError as err:
assert 'Failed to create AH' in err.args[0]
else:
raise PyverbsError('Created a non-global AH on RoCE')
def post_send(agr_obj, send_wr, gid_index, port):
"""
Post a single send WR to the QP. Post_send's second parameter (send bad wr)
is ignored for simplicity. For UD traffic an address vector is added as
well.
:param agr_obj: aggregation object which contains all resources necessary
:param send_wr: Send work request to post send
:param gid_index: Local gid index
:param port: IB port number
:return: None
"""
qp_type = agr_obj.qp.qp_type
if qp_type == e.IBV_QPT_UD:
gr = GlobalRoute(dgid=agr_obj.ctx.query_gid(port, gid_index),
sgid_index=gid_index)
ah_attr = AHAttr(port_num=port, is_global=1, gr=gr,
dlid=agr_obj.port_attr.lid)
ah = AH(agr_obj.pd, attr=ah_attr)
send_wr.set_wr_ud(ah, agr_obj.rqpn, agr_obj.UD_QKEY)
agr_obj.qp.post_send(send_wr, None)
def test_create_ah(self):
"""
Test ibv_create_ah.
"""
done = 0
for ctx, attr, attr_ex in self.devices:
pd = PD(ctx)
for port_num in range(1, 1 + attr.phys_port_cnt):
state = ctx.query_port(port_num).state
if state != e.IBV_PORT_ACTIVE and state != e.IBV_PORT_INIT:
continue
gr = u.get_global_route(ctx, port_num=port_num)
ah_attr = AHAttr(gr=gr, is_global=1, port_num=port_num)
try:
with AH(pd, attr=ah_attr):
done += 1
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create AH is not supported')
raise ex
if done == 0:
raise unittest.SkipTest('No port is up, can\'t create AH')