def active_side(dst_addr, syncer, notifier): """ RDMACM active side (client) which establish and uses rdamcm synchronous connection. :param dst_addr: Destination address to connect :param syncer: multiprocessing.Barrier object for processes synchronization :param notifier: Notify parent process about any exceptions or success :return: None """ try: client = CMResources(dst=dst_addr) syncer.wait() client.pre_run() connected_id = client.cmid client.create_mr(connected_id) send_msg = 'c' * client.msg_size for _ in range(client.num_msgs): client.mr.write(send_msg, client.msg_size) syncer.wait() connected_id.post_send(client.mr) connected_id.get_send_comp() syncer.wait() connected_id.post_recv(client.mr) syncer.wait() connected_id.get_recv_comp() msg_received = client.mr.read(client.msg_size, 0) validate(msg_received, False, client.msg_size) connected_id.disconnect() except Exception as ex: notifier.put('Caught exception in active side process: pid {}\n' .format(os.getpid()) + 'Exception message: {}'.format(str(ex))) else: notifier.put(None)
def passive_side(src_addr, syncer, notifier): """ RDMACM passive side (server) which establish and uses rdamcm synchronous connection. :param src_addr: Local address to bind to :param syncer: multiprocessing.Barrier object for processes synchronization :param notifier: Notify parent process about any exceptions or success :return: None """ try: server = CMResources(src=src_addr) server.pre_run() syncer.wait() connected_id = server.cmid.get_request() connected_id.accept() server.create_mr(connected_id) send_msg = 's' * server.msg_size for _ in range(server.num_msgs): connected_id.post_recv(server.mr) syncer.wait() syncer.wait() connected_id.get_recv_comp() msg_received = server.mr.read(server.msg_size, 0) validate(msg_received, True, server.msg_size) server.mr.write(send_msg, server.msg_size) connected_id.post_send(server.mr) connected_id.get_send_comp() syncer.wait() connected_id.disconnect() except Exception as ex: notifier.put('Caught exception in passive side process: pid {}\n' .format(os.getpid()) + 'Exception message: {}'.format(str(ex))) else: notifier.put(None)
def _cmid_client_traffic(self, multicast=False): """ RDMACM client side traffic function which sends and receives a message, and then validates the received message. This traffic method uses the RDMACM API for send, recv and get_completion. :return: None """ grh_offset = GRH_SIZE if self.cm_res.qp_type == e.IBV_QPT_UD else 0 send_msg = (self.cm_res.msg_size + grh_offset) * 'c' cmid = self.cm_res.cmid for _ in range(self.cm_res.num_msgs): self.cm_res.mr.write(send_msg, self.cm_res.msg_size + grh_offset) self.syncer.wait() if self.cm_res.port_space == ce.RDMA_PS_TCP: cmid.post_send(self.cm_res.mr) else: ah = AH(cmid.pd, attr=self.cm_res.ud_params.ah_attr) rqpn = MULTICAST_QPN if multicast else self.cm_res.ud_params.qp_num cmid.post_ud_send(self.cm_res.mr, ah, rqpn=rqpn, length=self.cm_res.msg_size) cmid.get_send_comp() cmid.post_recv(self.cm_res.mr) self.syncer.wait() self.syncer.wait() cmid.get_recv_comp() msg_received = self.cm_res.mr.read(self.cm_res.msg_size, grh_offset) validate(msg_received, False, self.cm_res.msg_size)
def remote_traffic(self, passive, remote_op='write'): """ Run rdmacm remote traffic. This method runs RDMA remote traffic from the active to the passive. :param passive: If True, run as server. :param remote_op: 'write'/'read', The type of the RDMA remote operation. """ msg_size = self.cm_res.msg_size if passive: self.cm_res.mr.write((msg_size) * 's', msg_size) mr_details = (self.cm_res.mr.rkey, self.cm_res.mr.buf) self.notifier.put(mr_details) self.syncer.wait() self.syncer.wait() if remote_op == 'write': msg_received = self.cm_res.mr.read(msg_size, 0) validate(msg_received, True, msg_size) else: self.cm_res.mr.write((msg_size) * 'c', msg_size) self.syncer.wait() rkey, remote_addr = self.notifier.get() cmid = self.cm_res.cmid post_func = cmid.post_write if remote_op == 'write' else \ cmid.post_read for _ in range(self.cm_res.num_msgs): post_func(self.cm_res.mr, msg_size, remote_addr, rkey, flags=e.IBV_SEND_SIGNALED) cmid.get_send_comp() self.syncer.wait() if remote_op == 'read': msg_received = self.cm_res.mr.read(msg_size, 0) validate(msg_received, False, msg_size)
def active_side(dst_addr, pipe): """ RDMACM active side (client) which establish and uses rdamcm synchronous connection. :param dst_addr: Destination address to connect :param pipe: multiprocessing.Pipe object for processes synchronization :return: None """ client = CMResources(dst=dst_addr) pipe.recv() client.pre_run() connected_id = client.cmid client.create_mr(connected_id) send_msg = 'c' * client.msg_size for _ in range(client.num_msgs): client.mr.write(send_msg, client.msg_size) pipe.recv() connected_id.post_send(client.mr) connected_id.get_send_comp() pipe.send('') connected_id.post_recv(client.mr) pipe.recv() connected_id.get_recv_comp() msg_received = client.mr.read(client.msg_size, 0) validate(msg_received, False, client.msg_size) connected_id.disconnect()
def mixed_traffic(self): s_recv_wr = u.get_recv_wr(self.server) u.post_recv(self.server, s_recv_wr) self.prepare_send_elements() for i in range(self.iters): self.client.qp.wr_start() if i % 2: self.client.mr.write('c' * self.client.mr.length, self.client.mr.length) self.client.qp.wr_flags = e.IBV_SEND_SIGNALED self.client.qp.wr_send() self.client.qp.wr_set_sge(self.regular_send_sge) else: self.client.mr.write('s' * self.client.mr.length, self.client.mr.length) self.client.qp.wr_raw_wqe(self.raw_send_wqe) self.client.qp.wr_complete() u.poll_cq_ex(self.client.cq) u.poll_cq_ex(self.server.cq) u.post_recv(self.server, s_recv_wr) if not i % 2 and self.client.cq.read_opcode() != e.IBV_WC_DRIVER2: raise PyverbsError( 'Opcode validation failed: expected ' f'{e.IBV_WC_DRIVER2}, received {self.client.cq.read_opcode()}' ) act_buffer = self.server.mr.read(self.server.mr.length, 0) u.validate(act_buffer, i % 2, self.server.mr.length)
def passive_side(src_addr, pipe): """ RDMACM passive side (server) which establish and uses rdamcm synchronous connection. :param src_addr: Local address to bind to :param pipe: multiprocessing.Pipe object for processes synchronization :return: None """ server = CMResources(src=src_addr) server.pre_run() pipe.send('') connected_id = server.cmid.get_request() connected_id.accept() server.create_mr(connected_id) send_msg = 's' * server.msg_size for _ in range(server.num_msgs): connected_id.post_recv(server.mr) pipe.send('') pipe.recv() connected_id.get_recv_comp() msg_received = server.mr.read(server.msg_size, 0) validate(msg_received, True, server.msg_size) server.mr.write(send_msg, server.msg_size) connected_id.post_send(server.mr) connected_id.get_send_comp() pipe.send('') connected_id.disconnect()
def client_traffic(agr_obj, syncer): """ RDMACM active side traffic function which sends and receives a message, and then validates the received message. This operation is executed <agr_obj.num_msgs> times. If agr_obj.with_ext_qp is set, the traffic will use the external QP (agr_obj.qp). :param agr_obj: Aggregation object which contains all necessary resources :param syncer: multiprocessing.Barrier object for processes synchronization :return: None """ if agr_obj.with_ext_qp: return _client_traffic_with_ext_qp(agr_obj, syncer) send_msg = agr_obj.msg_size * 'c' cmid = agr_obj.cmid for _ in range(agr_obj.num_msgs): agr_obj.mr.write(send_msg, agr_obj.msg_size) syncer.wait() cmid.post_send(agr_obj.mr) cmid.get_send_comp() syncer.wait() cmid.post_recv(agr_obj.mr) syncer.wait() cmid.get_recv_comp() msg_received = agr_obj.mr.read(agr_obj.msg_size, 0) validate(msg_received, agr_obj.is_server, agr_obj.msg_size)
def _client_traffic_with_ext_qp(agr_obj, syncer): recv_wr = get_recv_wr(agr_obj) syncer.wait() for _ in range(agr_obj.num_msgs): send_wr = get_send_element(agr_obj, agr_obj.is_server)[0] agr_obj.qp.post_send(send_wr) poll_cq(agr_obj.cq) agr_obj.qp.post_recv(recv_wr) poll_cq(agr_obj.cq) msg_received = agr_obj.mr.read(agr_obj.msg_size, 0) validate(msg_received, agr_obj.is_server, agr_obj.msg_size)
def _ext_qp_client_traffic(self): """ RDMACM client side traffic function which sends and receives a message, and then validates the received message. This traffic method uses the CM external QP and CQ for send, recv and get_completion. :return: None """ recv_wr = get_recv_wr(self.cm_res) self.syncer.wait() for _ in range(self.cm_res.num_msgs): send_wr = get_send_elements(self.cm_res, self.cm_res.passive)[0] self.cm_res.qp.post_send(send_wr) poll_cq(self.cm_res.cq) self.cm_res.qp.post_recv(recv_wr) poll_cq(self.cm_res.cq) msg_received = self.cm_res.mr.read(self.cm_res.msg_size, 0) validate(msg_received, self.cm_res.passive, self.cm_res.msg_size)
def server_traffic(agr_obj, syncer): """ RDMACM passive side traffic function which uses RDMACM's QP, this function sends and receives a message, and then validate the received message, this operation executed <agr_obj.num_msgs> times. :param agr_obj: Aggregation object which contains all necessary resources :param syncer: multiprocessing.Barrier object for processes synchronization :return: None """ send_msg = agr_obj.msg_size * 's' cmid = agr_obj.child_id for _ in range(agr_obj.num_msgs): cmid.post_recv(agr_obj.mr) syncer.wait() syncer.wait() cmid.get_recv_comp() msg_received = agr_obj.mr.read(agr_obj.msg_size, 0) validate(msg_received, agr_obj.is_server, agr_obj.msg_size) agr_obj.mr.write(send_msg, agr_obj.msg_size) cmid.post_send(agr_obj.mr) cmid.get_send_comp() syncer.wait()
def _cmid_server_traffic(self): """ RDMACM server side traffic function which sends and receives a message, and then validates the received message. This traffic method uses the RDMACM API for send, recv and get_completion. :return: None """ send_msg = (self.cm_res.msg_size) * 's' cmid = self.cm_res.child_id grh_offset = 0 for _ in range(self.cm_res.num_msgs): cmid.post_recv(self.cm_res.mr) self.syncer.wait() self.syncer.wait() wc = cmid.get_recv_comp() msg_received = self.cm_res.mr.read(self.cm_res.msg_size, grh_offset) validate(msg_received, self.cm_res.passive, self.cm_res.msg_size) self.cm_res.mr.write(send_msg, self.cm_res.msg_size) cmid.post_send(self.cm_res.mr) cmid.get_send_comp() self.syncer.wait()
def traffic_with_bad_flow(client, server, iters, gid_idx, port): """ Runs basic traffic with bad flow between two sides :param client: client side, clients base class is BaseTraffic :param server: server side, servers base class is BaseTraffic :param iters: number of traffic iterations :param gid_idx: local gid index :param port: IB port :return: None """ import tests.utils as u send_op = e.IBV_QP_EX_WITH_SEND ah_client = u.get_global_ah(client, gid_idx, port) s_recv_wr = u.get_recv_wr(server) c_recv_wr = u.get_recv_wr(client) for qp_idx in range(server.qp_count): # Prepare the receive queue with RecvWR u.post_recv(client, c_recv_wr, qp_idx=qp_idx) u.post_recv(server, s_recv_wr, qp_idx=qp_idx) read_offset = 0 for _ in range(iters): for qp_idx in range(server.qp_count): _, c_send_object = u.get_send_elements(client, False) u.send(client, c_send_object, send_op, True, qp_idx, ah_client, False) try: u.poll_cq(client.cq) except PyverbsError as ex: if client.bad_flow_handling(qp_idx, ex, True): continue raise ex u.poll_cq(server.cq) u.post_recv(server, s_recv_wr, qp_idx=qp_idx) msg_received = server.mr.read(server.msg_size, read_offset) u.validate(msg_received, True, server.msg_size) client.check_after_traffic()