def net_inference(self, msgs):
assert isinstance(msgs, pb.ForwardMsgs)
start = time.time()
img_batch = mx.nd.array(
np.zeros((self.batch_size, 3, self.width, self.height)))
for index, msg in enumerate(msgs.msgs):
arr = np.frombuffer(msg.network_input_buf,
dtype=np.float32).reshape(
(3, self.width, self.height))
img_batch[index] = arr
start_forward = time.time()
with self.lock:
self.mod.forward(Batch([img_batch]))
output_batch = self.mod.get_outputs()[0].asnumpy()
end_forward = time.time()
msgs_out = []
for i in range(len(msgs.msgs)):
msgs_out.append(
pb.ForwardMsg(network_output_buf=cPickle.dumps(
output_batch[i], protocol=cPickle.HIGHEST_PROTOCOL)))
log.info('{} use time {}, forward time {}, batch_size: {}/{}'.format(
self.app_name,
time.time() - start, end_forward - start_forward, len(msgs.msgs),
self.batch_size))
return pb.ForwardMsgs(msgs=msgs_out)
def run_test_tsv(times=1):
cases = parse_tsv_key(tsv_path)
srv = InferenceServer(const.app_name, const.cfg)
run_test_empty_image(srv)
run_test_bad_image(srv)
# run_test_bad_arg(srv)
for _ in range(times):
shuffle(cases)
for index, case in enumerate(cases):
start = time.time()
log.info('run case {}/{} image:{}'.format(index + 1, len(cases),
case[0]))
request = pb.InferenceRequest(data=pb.InferenceRequest.RequestData(
body=read_test_image_with_cache(tsv_prefix + case[0])),
params=json.dumps({'limit': 3}))
response = srv.net_inference_wrap(request)
assert isinstance(response, pb.InferenceResponse)
assert response.result
assert response.code == 200
result = json.loads(response.result)
confidences = result['confidences']
confidences.sort(key=lambda x: x['index'])
actual = json.loads(json.dumps(confidences))
expected = json.loads(case[1])
assert DeepDiff(expected, actual, significant_digits=3) == {}
log.info('use time {}'.format(time.time() - start))
def net_inference(self, msgs):
assert isinstance(msgs, pb.ForwardMsgs)
start = time.time()
start_forward = time.time()
assert len(msgs.msgs) == 1
arr = cPickle.loads(msgs.msgs[0].network_input_buf)
data = [[mx.nd.array(arr['im_array']), mx.nd.array(arr['im_info'])]]
data_batch = mx.io.DataBatch(data=data,
label=[None],
provide_data=arr['data_shapes'],
provide_label=[None])
with self.lock:
# https://github.com/ataraxialab/Deformable-ConvNets/blob/master/rfcn/core/tester.py#L124
scores, boxes, _ = im_detect(self.predictor, data_batch,
['data', 'im_info'], arr['im_scale'],
config)
end_forward = time.time()
msgs_out = []
for _ in range(len(msgs.msgs)):
output = {
'scores': scores[0],
'boxes': boxes[0],
}
msgs_out.append(
pb.ForwardMsg(network_output_buf=cPickle.dumps(
output, protocol=cPickle.HIGHEST_PROTOCOL)))
log.info('{} use time {}, forward time {}, batch_size: {}/{}'.format(
self.app_name,
time.time() - start, end_forward - start_forward, len(msgs.msgs),
self.batch_size))
return pb.ForwardMsgs(msgs=msgs_out)
def net_inference(self, msgs):
assert isinstance(msgs, pb.ForwardMsgs)
start = time.time()
start_forward = time.time()
for index, msg in enumerate(msgs.msgs):
r = cPickle.loads(msg.network_input_buf)
img_cls = r['img_cls']
assert img_cls.shape == (3, 225, 225)
img_det = r['img_det']
assert img_det.shape == (3, 320, 320)
self.net_fine.blobs['data'].data[index] = img_cls
self.net_coarse.blobs['data'].data[index] = img_cls
self.net_det.blobs['data'].data[index] = img_det
with self.lock:
output_fine = self.net_fine.forward()
output_coarse = self.net_coarse.forward()
output_det = self.net_det.forward()
assert output_fine['prob'].shape[1:] == (48, 1, 1)
# shape 第一维是 batch_size，第二维度是48类
assert output_coarse['prob'].shape[1:] == (7, 1, 1)
# shape 第一维是 batch_size，第二维度是7类
assert output_det['detection_out'].shape[1] == 1
assert output_det['detection_out'].shape[3] == 7
# shape 第一维是 batch_size，第三维度是检测到的物体数目，第四维度是类别
end_forward = time.time()
buf = cPickle.dumps(
{
'output_fine': output_fine,
'output_coarse': output_coarse,
'output_det': output_det
},
protocol=cPickle.HIGHEST_PROTOCOL)
msgs_out = []
for i in range(len(msgs.msgs)):
msgs_out.append(
pb.ForwardMsg(network_output_buf=buf,
meta={
"data":
json.dumps({
'image_index': i
}).encode('utf8')
}))
log.info('{} use time {}, forward time {}, batch_size: {}/{}'.format(
self.app_name,
time.time() - start, end_forward - start_forward, len(msgs.msgs),
self.batch_size))
return pb.ForwardMsgs(msgs=msgs_out)
def run_forward(self, serve_func):
n = self.forward_num
log.info("run_forward app_name:%s flavor:%s num:%s,", self.app_name,
self.flavor, n)
if n == 1:
t = threading.Lock()
serve_func(t)
elif n == 2:
process_lock = multiprocessing.Lock()
p1 = multiprocessing.Process(target=serve_func,
args=(process_lock, ))
p1.start()
p2 = multiprocessing.Process(target=serve_func,
args=(process_lock, ))
p2.start()
p1.join()
p2.join()
else:
raise Exception("unsupported")
def run_inference(self, serve_func):
n = self.inference_num
log.info("run_inference app_name:%s flavor:%s num:%s,", self.app_name,
self.flavor, n)
def serveOneProcess():
for _ in range(n.n_thread):
spawn(serve_func)
spawn(serve_func).join()
if n.n_process == 1:
serveOneProcess()
else:
ps = []
for _ in range(n.n_process):
p = multiprocessing.Process(target=serveOneProcess)
p.start()
ps.append(p)
for p in ps:
p.join()
def serve(self):
max_batch_size = self.batch_size
log.info('run forward max_batch_size:%s', max_batch_size)
network_in_context = zmq.Context()
network_in = network_in_context.socket(zmq.PULL)
network_in.connect(const.FORWARD_IN)
network_out_context = zmq.Context()
network_out = network_out_context.socket(zmq.PUSH)
network_out.connect(const.FORWARD_OUT)
inputs = []
self.monitor_push.send(
pb.MonitorMetric(
kind="forward_started_success",
pid=str(self.pid)).SerializeToString())
while True:
def process(buf):
msg = pb.ForwardMsg()
msg.ParseFromString(buf)
inputs.append(msg)
buf = network_in.recv()
process(buf)
while len(inputs) < max_batch_size:
try:
buf = network_in.recv(zmq.NOBLOCK)
process(buf)
except Again:
break
if not inputs:
continue
outputs = self.net_inference_wrap(
pb.ForwardMsgs(msgs=inputs[:max_batch_size]))
network_out.send(outputs.SerializeToString())
inputs = inputs[max_batch_size:]
import aisdk.common.config
from aisdk.common.flavor import Flavor, GPU_SHARE
from aisdk.common.logger import log
app_name = 'terror_mixup'
flavor = Flavor(app_name=app_name)
cfg = aisdk.common.config.load_config(app_name, flavor)
if flavor.flavor == GPU_SHARE:
log.info("flavor GPU_SHARE, change batch_size %s -> %s", cfg['batch_size'],
4)
cfg['batch_size'] = 4
