def GetJson(self, request, context):
meta = json.loads(request.meta)
img_path = request.buf.decode()
logger.info('Searching by ' + img_path)
# Decode image from buf
with StreamSuppressor():
results = self.search_machine.search(
image_query=img_path,
subtitle_query=meta['text'],
face_query=None,
topK=5,
tv_name=meta['target_videos'][0] if len(meta['target_videos']) else None
# TODO Currently only support one target video
)
# Convert tensor to list to make it serializable
for res in results:
# TODO Here has some bugs, json can not accept numpy results
if not type(res['FEATURE']) == list:
res['FEATURE'] = res['FEATURE'].tolist()
try:
if not type(res['AUDIO_FEATURE']) == list:
res['AUDIO_FEATURE'] = res['AUDIO_FEATURE'].tolist()
except:
pass
try:
if not type(res['SUBTITLE_FEATURE']) == list and not type(res['SUBTITLE_FEATURE']) == str:
res['SUBTITLE_FEATURE'] = res['SUBTITLE_FEATURE'].tolist()
except:
pass
return api2msl_pb2.JsonReply(json=json.dumps(results), meta='')
def GetJson(self, request, context):
img = cv2.imdecode(np.fromstring(request.buf, dtype=np.uint8), -1)
logger.info('Receiving image of shape ' + str(img.shape))
models = request.meta
models = models.split(',')
det = {}
if 'SSD-mobilenet' in models or 'SSD-inception' in models:
logger.info('Processing with SSD_inception')
det.update(self.detector_engine.detect(img))
if 'ctpn' in models:
logger.info('Processing with CTPN')
det.update(self.ctpn_engine.detect(img))
if 'mtcnn' in models:
logger.info('Processing with mtcnn')
boxes, name_list = self.face_engine.get_indentity(img, role=True)
boxes = np.array(boxes)
if boxes.size != 0:
# normalize box dimensions
img_shape = img.shape
boxes[:, (0, 2)] /= img_shape[1]
boxes[:, (1, 3)] /= img_shape[0]
# Swap x and y to make it consistent with other DNN APIs
boxes[:, (0, 1, 2, 3)] = boxes[:, (1, 0, 3, 2)]
det.update({
'face_bboxes': boxes.tolist(),
'face_names': name_list
})
if 'res18-places365' in models:
logger.info('Processing with res18-places365')
det.update(
self.places365_engine.detect(Image.fromarray(img),
tensor=True))
logger.info('Processing done')
return api2msl_pb2.JsonReply(json=json.dumps(det), meta='')
def GetJson(self, request, context):
res = {}
meta = request.meta
meta = meta.split(',')
# Process entire audio file
# Extract nlp feature from subtitle
if 'subtitle' in meta:
subtitle_path = request.buf.decode()
logger.info('Extracting from subtitle: ' + subtitle_path)
start_time = int(meta[1])
end_time = int(meta[2])
sentences = extract_srt(start_time, end_time, subtitle_path)
if len(sentences) == 0:
sentences_feature = 'unknown_feature'
sentences = 'unknown_subtitle'
else:
# TODO TEXT support what data types (BLOB only support numpy)
sentences = ' '.join(sentences)
sentences_feature = self.sentence_model.encode(sentences)
res['features'] = sentences_feature
return api2msl_pb2.JsonReply(json=json.dumps(res), meta=sentences)
# Extract audio feature
if 'audio' in meta:
audio_path = request.buf.decode()
logger.info('Extracting from audio: ' + audio_path)
start_time = int(meta[1])
end_time = int(meta[2])
audio_feature = self.audio_model.extract(audio_path, start_time,
end_time)[0]
res['features'] = audio_feature.tolist()
return api2msl_pb2.JsonReply(json=json.dumps(res), meta='')
if 'scene' in meta:
img = cv2.imdecode(np.fromstring(request.buf, dtype=np.uint8), -1)
logger.info('Extracting from image of shape ' + str(img.shape))
img_pil = Image.fromarray(img)
scene_feature = self.image_model.extract_vec(img_pil, True)
scene_name = self.image_model.detect(img_pil, True)
res['features'] = scene_feature.tolist()
return api2msl_pb2.JsonReply(json=json.dumps(res),
meta=scene_name['scene'][0])
return api2msl_pb2.JsonReply(json=json.dumps(res), meta='')
def GetJson(self, request, context):
res = {}
models = request.meta
models = models.split(',')
if 'soundnet' in models:
audio_path = request.buf.decode()
logger.info('Processing audio ' + audio_path)
with StreamSuppressor():
res = self.soundnet.classify_frame(audio_path, fr=FRAME_RATE)
logger.info('Finished Processing audio ' + audio_path)
return api2msl_pb2.JsonReply(json=json.dumps(res),
meta=str(FRAME_RATE))
def GetJson(self, request, context):
time_stamp = int(request.buf.decode())
video_path = request.meta
logger.info('Searching at ' + str(time_stamp) + ' in ' + video_path)
# Decode image from buf
with StreamSuppressor():
results = self.search_machine.search(time_stamp, video_path)
logger.info('Found ' + str(len(results)) + ' similar products')
# Convert tensor to list to make it serializable
for res in results:
# TODO Here has some bugs
if not type(res['FEATURE']) == list:
res['FEATURE'] = res['FEATURE'].tolist()
return api2msl_pb2.JsonReply(json=json.dumps(results), meta='')