def net_inference(self, msgs): assert isinstance(msgs, pb.ForwardMsgs) start = time.time() img_batch = mx.nd.array( np.zeros((self.batch_size, 3, self.width, self.height))) for index, msg in enumerate(msgs.msgs): arr = np.frombuffer(msg.network_input_buf, dtype=np.float32).reshape( (3, self.width, self.height)) img_batch[index] = arr start_forward = time.time() with self.lock: self.mod.forward(Batch([img_batch])) output_batch = self.mod.get_outputs()[0].asnumpy() end_forward = time.time() msgs_out = [] for i in range(len(msgs.msgs)): msgs_out.append( pb.ForwardMsg(network_output_buf=cPickle.dumps( output_batch[i], protocol=cPickle.HIGHEST_PROTOCOL))) log.info('{} use time {}, forward time {}, batch_size: {}/{}'.format( self.app_name, time.time() - start, end_forward - start_forward, len(msgs.msgs), self.batch_size)) return pb.ForwardMsgs(msgs=msgs_out)
def run_test_tsv(times=1): cases = parse_tsv_key(tsv_path) srv = InferenceServer(const.app_name, const.cfg) run_test_empty_image(srv) run_test_bad_image(srv) # run_test_bad_arg(srv) for _ in range(times): shuffle(cases) for index, case in enumerate(cases): start = time.time() log.info('run case {}/{} image:{}'.format(index + 1, len(cases), case[0])) request = pb.InferenceRequest(data=pb.InferenceRequest.RequestData( body=read_test_image_with_cache(tsv_prefix + case[0])), params=json.dumps({'limit': 3})) response = srv.net_inference_wrap(request) assert isinstance(response, pb.InferenceResponse) assert response.result assert response.code == 200 result = json.loads(response.result) confidences = result['confidences'] confidences.sort(key=lambda x: x['index']) actual = json.loads(json.dumps(confidences)) expected = json.loads(case[1]) assert DeepDiff(expected, actual, significant_digits=3) == {} log.info('use time {}'.format(time.time() - start))
def net_inference(self, msgs): assert isinstance(msgs, pb.ForwardMsgs) start = time.time() start_forward = time.time() assert len(msgs.msgs) == 1 arr = cPickle.loads(msgs.msgs[0].network_input_buf) data = [[mx.nd.array(arr['im_array']), mx.nd.array(arr['im_info'])]] data_batch = mx.io.DataBatch(data=data, label=[None], provide_data=arr['data_shapes'], provide_label=[None]) with self.lock: # https://github.com/ataraxialab/Deformable-ConvNets/blob/master/rfcn/core/tester.py#L124 scores, boxes, _ = im_detect(self.predictor, data_batch, ['data', 'im_info'], arr['im_scale'], config) end_forward = time.time() msgs_out = [] for _ in range(len(msgs.msgs)): output = { 'scores': scores[0], 'boxes': boxes[0], } msgs_out.append( pb.ForwardMsg(network_output_buf=cPickle.dumps( output, protocol=cPickle.HIGHEST_PROTOCOL))) log.info('{} use time {}, forward time {}, batch_size: {}/{}'.format( self.app_name, time.time() - start, end_forward - start_forward, len(msgs.msgs), self.batch_size)) return pb.ForwardMsgs(msgs=msgs_out)
def net_inference(self, msgs): assert isinstance(msgs, pb.ForwardMsgs) start = time.time() start_forward = time.time() for index, msg in enumerate(msgs.msgs): r = cPickle.loads(msg.network_input_buf) img_cls = r['img_cls'] assert img_cls.shape == (3, 225, 225) img_det = r['img_det'] assert img_det.shape == (3, 320, 320) self.net_fine.blobs['data'].data[index] = img_cls self.net_coarse.blobs['data'].data[index] = img_cls self.net_det.blobs['data'].data[index] = img_det with self.lock: output_fine = self.net_fine.forward() output_coarse = self.net_coarse.forward() output_det = self.net_det.forward() assert output_fine['prob'].shape[1:] == (48, 1, 1) # shape 第一维是 batch_size,第二维度是48类 assert output_coarse['prob'].shape[1:] == (7, 1, 1) # shape 第一维是 batch_size,第二维度是7类 assert output_det['detection_out'].shape[1] == 1 assert output_det['detection_out'].shape[3] == 7 # shape 第一维是 batch_size,第三维度是检测到的物体数目,第四维度是类别 end_forward = time.time() buf = cPickle.dumps( { 'output_fine': output_fine, 'output_coarse': output_coarse, 'output_det': output_det }, protocol=cPickle.HIGHEST_PROTOCOL) msgs_out = [] for i in range(len(msgs.msgs)): msgs_out.append( pb.ForwardMsg(network_output_buf=buf, meta={ "data": json.dumps({ 'image_index': i }).encode('utf8') })) log.info('{} use time {}, forward time {}, batch_size: {}/{}'.format( self.app_name, time.time() - start, end_forward - start_forward, len(msgs.msgs), self.batch_size)) return pb.ForwardMsgs(msgs=msgs_out)
def run_forward(self, serve_func): n = self.forward_num log.info("run_forward app_name:%s flavor:%s num:%s,", self.app_name, self.flavor, n) if n == 1: t = threading.Lock() serve_func(t) elif n == 2: process_lock = multiprocessing.Lock() p1 = multiprocessing.Process(target=serve_func, args=(process_lock, )) p1.start() p2 = multiprocessing.Process(target=serve_func, args=(process_lock, )) p2.start() p1.join() p2.join() else: raise Exception("unsupported")
def run_inference(self, serve_func): n = self.inference_num log.info("run_inference app_name:%s flavor:%s num:%s,", self.app_name, self.flavor, n) def serveOneProcess(): for _ in range(n.n_thread): spawn(serve_func) spawn(serve_func).join() if n.n_process == 1: serveOneProcess() else: ps = [] for _ in range(n.n_process): p = multiprocessing.Process(target=serveOneProcess) p.start() ps.append(p) for p in ps: p.join()
def serve(self): max_batch_size = self.batch_size log.info('run forward max_batch_size:%s', max_batch_size) network_in_context = zmq.Context() network_in = network_in_context.socket(zmq.PULL) network_in.connect(const.FORWARD_IN) network_out_context = zmq.Context() network_out = network_out_context.socket(zmq.PUSH) network_out.connect(const.FORWARD_OUT) inputs = [] self.monitor_push.send( pb.MonitorMetric( kind="forward_started_success", pid=str(self.pid)).SerializeToString()) while True: def process(buf): msg = pb.ForwardMsg() msg.ParseFromString(buf) inputs.append(msg) buf = network_in.recv() process(buf) while len(inputs) < max_batch_size: try: buf = network_in.recv(zmq.NOBLOCK) process(buf) except Again: break if not inputs: continue outputs = self.net_inference_wrap( pb.ForwardMsgs(msgs=inputs[:max_batch_size])) network_out.send(outputs.SerializeToString()) inputs = inputs[max_batch_size:]
import aisdk.common.config from aisdk.common.flavor import Flavor, GPU_SHARE from aisdk.common.logger import log app_name = 'terror_mixup' flavor = Flavor(app_name=app_name) cfg = aisdk.common.config.load_config(app_name, flavor) if flavor.flavor == GPU_SHARE: log.info("flavor GPU_SHARE, change batch_size %s -> %s", cfg['batch_size'], 4) cfg['batch_size'] = 4