def active_side(dst_addr, syncer, notifier):
"""
RDMACM active side (client) which establish and uses rdamcm synchronous
connection.
:param dst_addr: Destination address to connect
:param syncer: multiprocessing.Barrier object for processes synchronization
:param notifier: Notify parent process about any exceptions or success
:return: None
"""
try:
client = CMResources(dst=dst_addr)
syncer.wait()
client.pre_run()
connected_id = client.cmid
client.create_mr(connected_id)
send_msg = 'c' * client.msg_size
for _ in range(client.num_msgs):
client.mr.write(send_msg, client.msg_size)
syncer.wait()
connected_id.post_send(client.mr)
connected_id.get_send_comp()
syncer.wait()
connected_id.post_recv(client.mr)
syncer.wait()
connected_id.get_recv_comp()
msg_received = client.mr.read(client.msg_size, 0)
validate(msg_received, False, client.msg_size)
connected_id.disconnect()
except Exception as ex:
notifier.put('Caught exception in active side process: pid {}\n'
.format(os.getpid()) +
'Exception message: {}'.format(str(ex)))
else:
notifier.put(None)
def passive_side(src_addr, syncer, notifier):
"""
RDMACM passive side (server) which establish and uses rdamcm synchronous
connection.
:param src_addr: Local address to bind to
:param syncer: multiprocessing.Barrier object for processes synchronization
:param notifier: Notify parent process about any exceptions or success
:return: None
"""
try:
server = CMResources(src=src_addr)
server.pre_run()
syncer.wait()
connected_id = server.cmid.get_request()
connected_id.accept()
server.create_mr(connected_id)
send_msg = 's' * server.msg_size
for _ in range(server.num_msgs):
connected_id.post_recv(server.mr)
syncer.wait()
syncer.wait()
connected_id.get_recv_comp()
msg_received = server.mr.read(server.msg_size, 0)
validate(msg_received, True, server.msg_size)
server.mr.write(send_msg, server.msg_size)
connected_id.post_send(server.mr)
connected_id.get_send_comp()
syncer.wait()
connected_id.disconnect()
except Exception as ex:
notifier.put('Caught exception in passive side process: pid {}\n'
.format(os.getpid()) +
'Exception message: {}'.format(str(ex)))
else:
notifier.put(None)
def _cmid_client_traffic(self, multicast=False):
"""
RDMACM client side traffic function which sends and receives a message,
and then validates the received message. This traffic method uses the
RDMACM API for send, recv and get_completion.
:return: None
"""
grh_offset = GRH_SIZE if self.cm_res.qp_type == e.IBV_QPT_UD else 0
send_msg = (self.cm_res.msg_size + grh_offset) * 'c'
cmid = self.cm_res.cmid
for _ in range(self.cm_res.num_msgs):
self.cm_res.mr.write(send_msg, self.cm_res.msg_size + grh_offset)
self.syncer.wait()
if self.cm_res.port_space == ce.RDMA_PS_TCP:
cmid.post_send(self.cm_res.mr)
else:
ah = AH(cmid.pd, attr=self.cm_res.ud_params.ah_attr)
rqpn = MULTICAST_QPN if multicast else self.cm_res.ud_params.qp_num
cmid.post_ud_send(self.cm_res.mr,
ah,
rqpn=rqpn,
length=self.cm_res.msg_size)
cmid.get_send_comp()
cmid.post_recv(self.cm_res.mr)
self.syncer.wait()
self.syncer.wait()
cmid.get_recv_comp()
msg_received = self.cm_res.mr.read(self.cm_res.msg_size,
grh_offset)
validate(msg_received, False, self.cm_res.msg_size)
def remote_traffic(self, passive, remote_op='write'):
"""
Run rdmacm remote traffic. This method runs RDMA remote traffic from
the active to the passive.
:param passive: If True, run as server.
:param remote_op: 'write'/'read', The type of the RDMA remote operation.
"""
msg_size = self.cm_res.msg_size
if passive:
self.cm_res.mr.write((msg_size) * 's', msg_size)
mr_details = (self.cm_res.mr.rkey, self.cm_res.mr.buf)
self.notifier.put(mr_details)
self.syncer.wait()
self.syncer.wait()
if remote_op == 'write':
msg_received = self.cm_res.mr.read(msg_size, 0)
validate(msg_received, True, msg_size)
else:
self.cm_res.mr.write((msg_size) * 'c', msg_size)
self.syncer.wait()
rkey, remote_addr = self.notifier.get()
cmid = self.cm_res.cmid
post_func = cmid.post_write if remote_op == 'write' else \
cmid.post_read
for _ in range(self.cm_res.num_msgs):
post_func(self.cm_res.mr,
msg_size,
remote_addr,
rkey,
flags=e.IBV_SEND_SIGNALED)
cmid.get_send_comp()
self.syncer.wait()
if remote_op == 'read':
msg_received = self.cm_res.mr.read(msg_size, 0)
validate(msg_received, False, msg_size)
def active_side(dst_addr, pipe):
"""
RDMACM active side (client) which establish and uses rdamcm synchronous
connection.
:param dst_addr: Destination address to connect
:param pipe: multiprocessing.Pipe object for processes synchronization
:return: None
"""
client = CMResources(dst=dst_addr)
pipe.recv()
client.pre_run()
connected_id = client.cmid
client.create_mr(connected_id)
send_msg = 'c' * client.msg_size
for _ in range(client.num_msgs):
client.mr.write(send_msg, client.msg_size)
pipe.recv()
connected_id.post_send(client.mr)
connected_id.get_send_comp()
pipe.send('')
connected_id.post_recv(client.mr)
pipe.recv()
connected_id.get_recv_comp()
msg_received = client.mr.read(client.msg_size, 0)
validate(msg_received, False, client.msg_size)
connected_id.disconnect()
def mixed_traffic(self):
s_recv_wr = u.get_recv_wr(self.server)
u.post_recv(self.server, s_recv_wr)
self.prepare_send_elements()
for i in range(self.iters):
self.client.qp.wr_start()
if i % 2:
self.client.mr.write('c' * self.client.mr.length,
self.client.mr.length)
self.client.qp.wr_flags = e.IBV_SEND_SIGNALED
self.client.qp.wr_send()
self.client.qp.wr_set_sge(self.regular_send_sge)
else:
self.client.mr.write('s' * self.client.mr.length,
self.client.mr.length)
self.client.qp.wr_raw_wqe(self.raw_send_wqe)
self.client.qp.wr_complete()
u.poll_cq_ex(self.client.cq)
u.poll_cq_ex(self.server.cq)
u.post_recv(self.server, s_recv_wr)
if not i % 2 and self.client.cq.read_opcode() != e.IBV_WC_DRIVER2:
raise PyverbsError(
'Opcode validation failed: expected '
f'{e.IBV_WC_DRIVER2}, received {self.client.cq.read_opcode()}'
)
act_buffer = self.server.mr.read(self.server.mr.length, 0)
u.validate(act_buffer, i % 2, self.server.mr.length)
def passive_side(src_addr, pipe):
"""
RDMACM passive side (server) which establish and uses rdamcm synchronous
connection.
:param src_addr: Local address to bind to
:param pipe: multiprocessing.Pipe object for processes synchronization
:return: None
"""
server = CMResources(src=src_addr)
server.pre_run()
pipe.send('')
connected_id = server.cmid.get_request()
connected_id.accept()
server.create_mr(connected_id)
send_msg = 's' * server.msg_size
for _ in range(server.num_msgs):
connected_id.post_recv(server.mr)
pipe.send('')
pipe.recv()
connected_id.get_recv_comp()
msg_received = server.mr.read(server.msg_size, 0)
validate(msg_received, True, server.msg_size)
server.mr.write(send_msg, server.msg_size)
connected_id.post_send(server.mr)
connected_id.get_send_comp()
pipe.send('')
connected_id.disconnect()
def client_traffic(agr_obj, syncer):
"""
RDMACM active side traffic function which sends and receives a message, and
then validates the received message. This operation is executed
<agr_obj.num_msgs> times. If agr_obj.with_ext_qp is set, the traffic will
use the external QP (agr_obj.qp).
:param agr_obj: Aggregation object which contains all necessary resources
:param syncer: multiprocessing.Barrier object for processes synchronization
:return: None
"""
if agr_obj.with_ext_qp:
return _client_traffic_with_ext_qp(agr_obj, syncer)
send_msg = agr_obj.msg_size * 'c'
cmid = agr_obj.cmid
for _ in range(agr_obj.num_msgs):
agr_obj.mr.write(send_msg, agr_obj.msg_size)
syncer.wait()
cmid.post_send(agr_obj.mr)
cmid.get_send_comp()
syncer.wait()
cmid.post_recv(agr_obj.mr)
syncer.wait()
cmid.get_recv_comp()
msg_received = agr_obj.mr.read(agr_obj.msg_size, 0)
validate(msg_received, agr_obj.is_server, agr_obj.msg_size)
def _client_traffic_with_ext_qp(agr_obj, syncer):
recv_wr = get_recv_wr(agr_obj)
syncer.wait()
for _ in range(agr_obj.num_msgs):
send_wr = get_send_element(agr_obj, agr_obj.is_server)[0]
agr_obj.qp.post_send(send_wr)
poll_cq(agr_obj.cq)
agr_obj.qp.post_recv(recv_wr)
poll_cq(agr_obj.cq)
msg_received = agr_obj.mr.read(agr_obj.msg_size, 0)
validate(msg_received, agr_obj.is_server, agr_obj.msg_size)
def _ext_qp_client_traffic(self):
"""
RDMACM client side traffic function which sends and receives a message,
and then validates the received message. This traffic method uses the CM
external QP and CQ for send, recv and get_completion.
:return: None
"""
recv_wr = get_recv_wr(self.cm_res)
self.syncer.wait()
for _ in range(self.cm_res.num_msgs):
send_wr = get_send_elements(self.cm_res, self.cm_res.passive)[0]
self.cm_res.qp.post_send(send_wr)
poll_cq(self.cm_res.cq)
self.cm_res.qp.post_recv(recv_wr)
poll_cq(self.cm_res.cq)
msg_received = self.cm_res.mr.read(self.cm_res.msg_size, 0)
validate(msg_received, self.cm_res.passive, self.cm_res.msg_size)
def server_traffic(agr_obj, syncer):
"""
RDMACM passive side traffic function which uses RDMACM's QP, this function
sends and receives a message, and then validate the received message, this
operation executed <agr_obj.num_msgs> times.
:param agr_obj: Aggregation object which contains all necessary resources
:param syncer: multiprocessing.Barrier object for processes synchronization
:return: None
"""
send_msg = agr_obj.msg_size * 's'
cmid = agr_obj.child_id
for _ in range(agr_obj.num_msgs):
cmid.post_recv(agr_obj.mr)
syncer.wait()
syncer.wait()
cmid.get_recv_comp()
msg_received = agr_obj.mr.read(agr_obj.msg_size, 0)
validate(msg_received, agr_obj.is_server, agr_obj.msg_size)
agr_obj.mr.write(send_msg, agr_obj.msg_size)
cmid.post_send(agr_obj.mr)
cmid.get_send_comp()
syncer.wait()
def _cmid_server_traffic(self):
"""
RDMACM server side traffic function which sends and receives a message,
and then validates the received message. This traffic method uses the
RDMACM API for send, recv and get_completion.
:return: None
"""
send_msg = (self.cm_res.msg_size) * 's'
cmid = self.cm_res.child_id
grh_offset = 0
for _ in range(self.cm_res.num_msgs):
cmid.post_recv(self.cm_res.mr)
self.syncer.wait()
self.syncer.wait()
wc = cmid.get_recv_comp()
msg_received = self.cm_res.mr.read(self.cm_res.msg_size,
grh_offset)
validate(msg_received, self.cm_res.passive, self.cm_res.msg_size)
self.cm_res.mr.write(send_msg, self.cm_res.msg_size)
cmid.post_send(self.cm_res.mr)
cmid.get_send_comp()
self.syncer.wait()
def traffic_with_bad_flow(client, server, iters, gid_idx, port):
"""
Runs basic traffic with bad flow between two sides
:param client: client side, clients base class is BaseTraffic
:param server: server side, servers base class is BaseTraffic
:param iters: number of traffic iterations
:param gid_idx: local gid index
:param port: IB port
:return: None
"""
import tests.utils as u
send_op = e.IBV_QP_EX_WITH_SEND
ah_client = u.get_global_ah(client, gid_idx, port)
s_recv_wr = u.get_recv_wr(server)
c_recv_wr = u.get_recv_wr(client)
for qp_idx in range(server.qp_count):
# Prepare the receive queue with RecvWR
u.post_recv(client, c_recv_wr, qp_idx=qp_idx)
u.post_recv(server, s_recv_wr, qp_idx=qp_idx)
read_offset = 0
for _ in range(iters):
for qp_idx in range(server.qp_count):
_, c_send_object = u.get_send_elements(client, False)
u.send(client, c_send_object, send_op, True, qp_idx, ah_client,
False)
try:
u.poll_cq(client.cq)
except PyverbsError as ex:
if client.bad_flow_handling(qp_idx, ex, True):
continue
raise ex
u.poll_cq(server.cq)
u.post_recv(server, s_recv_wr, qp_idx=qp_idx)
msg_received = server.mr.read(server.msg_size, read_offset)
u.validate(msg_received, True, server.msg_size)
client.check_after_traffic()
