def fcis_seg(image, classes, predictor, args): num_classes = len(classes) + 1 data = [] target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(image, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) start = time.time() im_info = np.array([[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data.append({'data': im_tensor, 'im_info': im_info}) data_names = ['data', 'im_info'] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[0])]], provide_label=[None]) scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))] scores, boxes, masks, data_dict = im_detect(predictor, data_batch, data_names, scales, config) im_shapes = [data_batch.data[i][0].shape[2:4] for i in xrange(len(data_batch.data))] if not config.TEST.USE_MASK_MERGE: all_boxes = [[] for _ in xrange(num_classes)] all_masks = [[] for _ in xrange(num_classes)] nms = py_nms_wrapper(config.TEST.NMS) for j in range(1, num_classes): indexes = np.where(scores[0][:, j] > 0.7)[0] cls_scores = scores[0][indexes, j, np.newaxis] cls_masks = masks[0][indexes, 1, :, :] try: if config.CLASS_AGNOSTIC: cls_boxes = boxes[0][indexes, :] else: raise Exception() except: cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) all_boxes[j] = cls_dets[keep, :] all_masks[j] = cls_masks[keep, :] dets = [all_boxes[j] for j in range(1, num_classes)] masks = [all_masks[j] for j in range(1, num_classes)] else: masks = masks[0][:, 1:, :, :] im_height = np.round(im_shapes[0][0] / scales[0]).astype('int') im_width = np.round(im_shapes[0][1] / scales[0]).astype('int') # print (im_height, im_width) boxes = clip_boxes(boxes[0], (im_height, im_width)) result_masks, result_dets = gpu_mask_voting(masks, boxes, scores[0], num_classes, 100, im_width, im_height, config.TEST.NMS, config.TEST.MASK_MERGE_THRESH, config.BINARY_THRESH, 0) dets = [result_dets[j] for j in range(1, num_classes)] masks = [result_masks[j][:, 0, :, :] for j in range(1, num_classes)] cods, bimsks, names = decode_mask(im, dets, masks, classes, config, args) return cods, bimsks, names
def inference(predictor, data_batch, data_names, num_classes, BINARY_THRESH = 0.4, CONF_THRESH=0.7, gpu_id=0): scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))] im_shapes = [data_batch.data[i][0].shape[2:4] for i in xrange(len(data_batch.data))] scores, boxes, masks, data_dict = im_detect(predictor, data_batch, data_names, scales, config) if not config.TEST.USE_MASK_MERGE: all_boxes = [[] for _ in xrange(num_classes)] all_masks = [[] for _ in xrange(num_classes)] nms = py_nms_wrapper(config.TEST.NMS) for j in range(1, num_classes): indexes = np.where(scores[0][:, j] > CONF_THRESH)[0] cls_scores = scores[0][indexes, j, np.newaxis] cls_masks = masks[0][indexes, 1, :, :] # try: # if config.CLASS_AGNOSTIC: # cls_boxes = boxes[0][indexes, :] # else: # raise Exception() # except: if config.CLASS_AGNOSTIC: cls_boxes = boxes[0][indexes, :] else: cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) all_boxes[j] = cls_dets[keep, :] all_masks[j] = cls_masks[keep, :] dets = [all_boxes[j] for j in range(1, num_classes)] masks = [all_masks[j] for j in range(1, num_classes)] else: masks = masks[0][:, 1:, :, :] im_height = np.round(im_shapes[0][0] / scales[0]).astype('int') im_width = np.round(im_shapes[0][1] / scales[0]).astype('int') # print (im_height, im_width) boxes = clip_boxes(boxes[0], (im_height, im_width)) result_masks, result_dets = gpu_mask_voting(masks, boxes, scores[0], num_classes, 100, im_width, im_height, config.TEST.NMS, config.TEST.MASK_MERGE_THRESH, BINARY_THRESH, gpu_id) dets = [result_dets[j] for j in range(1, num_classes)] masks = [result_masks[j][:, 0, :, :] for j in range(1, num_classes)] for i in xrange(len(dets)): keep = np.where(dets[i][:,-1] > CONF_THRESH) dets[i] = dets[i][keep] masks[i] = masks[i][keep] return dets, masks
def main(): # get symbol ctx_id = [int(i) for i in config.gpus.split(',')] pprint.pprint(config) sym_instance = eval(config.symbol)() sym = sym_instance.get_symbol(config, is_train=False) # set up class names # set up class names num_classes = 81 classes = [ 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush' ] # load demo data image_names = [] names_dirs = os.listdir(cur_path + '/../' + test_dir) for im_name in names_dirs: if im_name[-4:] == '.jpg' or im_name[-4:] == '.png': image_names.append(im_name) data = [] for im_name in image_names: assert os.path.exists(cur_path + '/../' + test_dir + im_name), ( '%s does not exist'.format('../demo/' + im_name)) im = cv2.imread(cur_path + '/../' + test_dir + im_name, cv2.IMREAD_COLOR | long(128)) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] #print "before scale: " #print im.shape im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) #print "after scale: " #print im.shape #im_scale = 1.0 #print "scale ratio: " #print im_scale im_tensor = transform(im, config.network.PIXEL_MEANS) #print im_tensor.shape im_info = np.array( [[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data.append({'data': im_tensor, 'im_info': im_info}) # get predictor data_names = ['data', 'im_info'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))] provide_label = [None for i in xrange(len(data))] arg_params, aux_params = load_param(cur_path + '/../' + model_dir, 0, process=True) predictor = Predictor(sym, data_names, label_names, context=[mx.gpu(ctx_id[0])], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) # warm up for i in xrange(2): data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0, provide_data=[[ (k, v.shape) for k, v in zip(data_names, data[0]) ]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] _, _, _, _ = im_detect(predictor, data_batch, data_names, scales, config) # test for idx, im_name in enumerate(image_names): data_batch = mx.io.DataBatch( data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] tic() scores, boxes, masks, data_dict = im_detect(predictor, data_batch, data_names, [1.0], config) im_shapes = [ data_batch.data[i][0].shape[2:4] for i in xrange(len(data_batch.data)) ] #print im_shapes if not config.TEST.USE_MASK_MERGE: all_boxes = [[] for _ in xrange(num_classes)] all_masks = [[] for _ in xrange(num_classes)] nms = py_nms_wrapper(config.TEST.NMS) for j in range(1, num_classes): indexes = np.where(scores[0][:, j] > 0.7)[0] cls_scores = scores[0][indexes, j, np.newaxis] cls_masks = masks[0][indexes, 1, :, :] try: if config.CLASS_AGNOSTIC: cls_boxes = boxes[0][indexes, :] else: raise Exception() except: cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) all_boxes[j] = cls_dets[keep, :] all_masks[j] = cls_masks[keep, :] dets = [all_boxes[j] for j in range(1, num_classes)] masks = [all_masks[j] for j in range(1, num_classes)] else: masks = masks[0][:, 1:, :, :] result_masks, result_dets = gpu_mask_voting( masks, boxes[0], scores[0], num_classes, 100, im_shapes[0][1], im_shapes[0][0], config.TEST.NMS, config.TEST.MASK_MERGE_THRESH, config.BINARY_THRESH, ctx_id[0]) dets = [result_dets[j] for j in range(1, num_classes)] masks = [ result_masks[j][:, 0, :, :] for j in range(1, num_classes) ] print 'testing {} {:.4f}s'.format(im_name, toc()) # visualize for i in xrange(len(dets)): keep = np.where(dets[i][:, -1] > 0.7) dets[i] = dets[i][keep] masks[i] = masks[i][keep] im = cv2.imread(cur_path + '/../' + test_dir + im_name) im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) show_masks(im, dets, masks, classes, config, 1.0 / scales[0], False) # Save img cv2.imwrite(cur_path + '/../' + result_dir + im_name, cv2.cvtColor(im, cv2.COLOR_BGR2RGB)) print 'done'
def forward(self, multiple=True, gt_dir=None): """ :param multiple: :param gt_dir: :return: """ self.multiple = multiple # if multiple is False, gt_dir must be provided if not self.multiple: assert gt_dir is not None for idx, itm in enumerate(self.batch_list): itm = itm.split('/')[-1] data_batch = mx.io.DataBatch( data=[self.data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(self.data_names, self.data[idx]) ]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] im_shapes = [ data_batch.data[i][0].shape[2:4] for i in xrange(len(data_batch.data)) ] tic() scores, boxes, masks, convs, data_dict = conv_detect( self.predictor, data_batch, self.data_names, scales, config) im_height = np.round(im_shapes[0][0] / scales[0]).astype('int') im_width = np.round(im_shapes[0][1] / scales[0]).astype('int') # return # (1) mask merge if not config.TEST.USE_MASK_MERGE: all_boxes = [[] for _ in xrange(self.num_classes)] all_masks = [[] for _ in xrange(self.num_classes)] nms = py_nms_wrapper(config.TEST.NMS) for j in range(1, self.num_classes): indexes = np.where(scores[0][:, j] > 0.7)[0] cls_scores = scores[0][indexes, j, np.newaxis] cls_masks = masks[0][indexes, 1, :, :] try: if config.CLASS_AGNOSTIC: cls_boxes = boxes[0][indexes, :] else: raise Exception() except: cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) all_boxes[j] = cls_dets[keep, :] all_masks[j] = cls_masks[keep, :] dets = [all_boxes[j] for j in range(1, self.num_classes)] masks = [all_masks[j] for j in range(1, self.num_classes)] else: masks = masks[0][:, 1:, :, :] boxes = clip_boxes(boxes[0], (im_height, im_width)) # gpu mask voting if not self.cpu_mask_vote: result_masks, result_dets = gpu_mask_voting( masks, boxes, scores[0], self.num_classes, 100, im_width, im_height, config.TEST.NMS, config.TEST.MASK_MERGE_THRESH, config.BINARY_THRESH, self.ctx[0]) # cpu mask voting else: result_masks, result_dets = cpu_mask_voting( masks, boxes, scores[0], self.num_classes, 100, im_width, im_height, config.TEST.NMS, config.TEST.MASK_MERGE_THRESH, config.BINARY_THRESH) # dets represent coordinates of bounding-boxes(up left, bottom right) dets = [result_dets[j] for j in range(1, self.num_classes)] masks = [ result_masks[j][:, 0, :, :] for j in range(1, self.num_classes) ] # (2) filter the result whose detection probability is under 0.7 for i in xrange(len(dets)): keep = np.where(dets[i][:, -1] > 0.7) dets[i] = dets[i][keep] masks[i] = masks[i][keep] # (3) prepare for roi-pooling roi = [] # scaled bounding box coordinates(up left, bottom right) info = [ ] # class label, name of instance, probability, bounding box coordinates(up left, bottom right) if self.multiple: for k in xrange(0, self.num_classes - 1): nums = len(dets[k]) if nums > 0: for j in xrange(nums): roi.append(dets[k][j][0:-1] * scales[0] ) # note that the input image is scaled info.append((k, itm[:-4] + '-' + self.classes[k] + '_' + str(j + 1), dets[k][j][-1], np.array(np.round(dets[k][j][0:-1])))) else: # Method 1 dist = [] temp_roi = [] temp_info = [] # Instance fn = itm[:-4].split('_')[0] # Instre # fn = itm[:-4] with open(os.path.join(gt_dir, fn + '.txt'), 'r') as strm: cod = strm.readline() if cod.split('\t')[-1] != "\n": # Instance cod = [int(ele) for ele in cod.split('\t')[1:]] else: # Instre cod = [int(ele) for ele in cod.split('\t')[1:-1]] cod_center_x = cod[0] + cod[2] / 2 cod_center_y = cod[1] + cod[3] / 2 for k in xrange(0, self.num_classes - 1): nums = len(dets[k]) if nums > 0: for j in range(nums): det_center_x = ( (dets[k][j][2] - dets[k][j][0]) / 2 + dets[k][j][0]) det_center_y = ( (dets[k][j][3] - dets[k][j][1]) / 2 + dets[k][j][1]) x = (det_center_x - cod_center_x) * (det_center_x - cod_center_x) y = (det_center_y - cod_center_y) * (det_center_y - cod_center_y) dist.append(np.sqrt(x + y)) temp_roi.append( dets[k][j][0:-1] * scales[0] ) # note that the input image is scaled temp_info.append( (k, itm[:-4] + '-' + self.classes[k] + '_' + str(j + 1), dets[k][j][-1], np.array(np.round(dets[k][j][0:-1])))) npdist = np.array(dist) order = np.argsort(npdist) if len(order) > 0: roi.append(temp_roi[order[0]]) info.append(temp_info[order[0]]) # Method 2 # cod[2] += cod[0] # cod[3] += cod[1] # cords = np.array(cod) # roi.append(cords * scales[0]) # info.append((0, itm[:-4] + '-' + self.classes[0] + '_' + str(0 + 1), 0, np.array(cod))) # (4) perform roi-pooling & output features = pooling_delegator(convs, roi, self.pool_size, self.ctx[0], config) """ pca """ if features is not None and not self.mapping: features = self.pca.transform(features) features = normalize(features, norm='l2', axis=1) if self.img is not None and self.ftr is not None and features is not None: self.output(info, features) print 'testing {} {:.4f}s'.format(itm, toc()) # (5) visualize & save """""" if self.dst_dir: pass # im = cv2.imread(self.src_dir + itm) # im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) # im = show_masks(im, dets, masks, self.classes, config) # im = save_results(self.dst_dir + itm.replace('.jpg', '.dtc.jpg'), im, dets, masks, self.classes, config) # cv2.imwrite(self.dst_dir + itm.replace('.jpg', '.msk.jpg'), im) """"""
def main(): # get symbol ctx_id = [int(i) for i in config.gpus.split(',')] pprint.pprint(config) sym_instance = eval(config.symbol)() sym = sym_instance.get_symbol(config, is_train=False) # set up class names num_classes = 31 classes =['alphabet_of_the_magi', 'anglo-saxon_futhorc', 'arcadian', 'armenian', 'asomtavruli_(georgian)', 'balinese', 'bengali', 'blackfoot_(canadian_aboriginal_syllabics)', 'braille', 'burmese_(myanmar)', 'cyrillic', 'early_aramaic', 'futurama', 'grantha', 'greek', 'gujarati', 'hebrew', 'inuktitut_(canadian_aboriginal_syllabics)', 'japanese_(hiragana)', 'japanese_(katakana)', 'korean', 'latin', 'malay_(jawi_-_arabic)', 'mkhedruli_(georgian)', 'n_ko', 'ojibwe_(canadian_aboriginal_syllabics)', 'sanskrit', 'syriac_(estrangelo)', 'tagalog', 'tifinagh'] # load demo data image_names = ['train_000000.jpg', 'train_000001.jpg', 'train_000002.jpg', 'val_000000.jpg', 'val_000001.jpg', 'val_000002.jpg'] data = [] for im_name in image_names: assert os.path.exists(cur_path + '/../demo/' + im_name), ('%s does not exist'.format('../demo/' + im_name)) im = cv2.imread(cur_path + '/../demo/' + im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array([[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data.append({'data': im_tensor, 'im_info': im_info}) # get predictor data_names = ['data', 'im_info'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))] provide_label = [None for i in xrange(len(data))] arg_params, aux_params = load_param(cur_path + '/../model/fcis_voc', 0, process=True) predictor = Predictor(sym, data_names, label_names, context=[mx.gpu(ctx_id[0])], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) # warm up for i in xrange(2): data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[0])]], provide_label=[None]) scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))] _, _, _, _ = im_detect(predictor, data_batch, data_names, scales, config) # test for idx, im_name in enumerate(image_names): data_batch = mx.io.DataBatch(data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))] tic() scores, boxes, masks, data_dict = im_detect(predictor, data_batch, data_names, scales, config) im_shapes = [data_batch.data[i][0].shape[2:4] for i in xrange(len(data_batch.data))] if not config.TEST.USE_MASK_MERGE: all_boxes = [[] for _ in xrange(num_classes)] all_masks = [[] for _ in xrange(num_classes)] nms = py_nms_wrapper(config.TEST.NMS) for j in range(1, num_classes): indexes = np.where(scores[0][:, j] > 0.7)[0] cls_scores = scores[0][indexes, j, np.newaxis] cls_masks = masks[0][indexes, 1, :, :] try: if config.CLASS_AGNOSTIC: cls_boxes = boxes[0][indexes, :] else: raise Exception() except: cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) all_boxes[j] = cls_dets[keep, :] all_masks[j] = cls_masks[keep, :] dets = [all_boxes[j] for j in range(1, num_classes)] masks = [all_masks[j] for j in range(1, num_classes)] else: masks = masks[0][:, 1:, :, :] im_height = np.round(im_shapes[0][0] / scales[0]).astype('int') im_width = np.round(im_shapes[0][1] / scales[0]).astype('int') print (im_height, im_width) boxes = clip_boxes(boxes[0], (im_height, im_width)) result_masks, result_dets = gpu_mask_voting(masks, boxes, scores[0], num_classes, 100, im_width, im_height, config.TEST.NMS, config.TEST.MASK_MERGE_THRESH, config.BINARY_THRESH, ctx_id[0]) dets = [result_dets[j] for j in range(1, num_classes)] masks = [result_masks[j][:, 0, :, :] for j in range(1, num_classes)] print 'testing {} {:.4f}s'.format(im_name, toc()) # visualize for i in xrange(len(dets)): keep = np.where(dets[i][:,-1]>0.7) dets[i] = dets[i][keep] masks[i] = masks[i][keep] im = cv2.imread(cur_path + '/../demo/' + im_name) im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) show_masks(im, dets, masks, classes, config) print 'done'
def main(): # get symbol ctx_id = [int(i) for i in config.gpus.split(',')] pprint.pprint(config) sym_instance = eval(config.symbol)() sym = sym_instance.get_symbol(config, is_train=False) # set up class names num_classes = 6 classes = [ '__background__', # always index 0 '1', '2', '3', '4', '5' ] # classes = ['__background__', # always index 0 # 'CubeBody', 'CubeFace', 'CylinderBody', 'CylinderFace', 'Grip'] # load demo data image_names = [] names_dirs = os.listdir(cur_path + '/../' + test_dir) for im_name in names_dirs: if im_name[-4:] == '.jpg': image_names.append(im_name) data = [] # next [3001: len(image_names)] for im_name in image_names[9001:len(image_names)]: assert os.path.exists(cur_path + '/../' + test_dir + im_name), ( '%s does not exist'.format('../demo/' + im_name)) im = cv2.imread(cur_path + '/../' + test_dir + im_name, cv2.IMREAD_COLOR | long(128)) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] # print "before scale: " # print im.shape im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) # print "after scale: " # print im.shape # im_scale = 1.0 # print "scale ratio: " # print im_scale im_tensor = transform(im, config.network.PIXEL_MEANS) # print im_tensor.shape im_info = np.array( [[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data.append({'data': im_tensor, 'im_info': im_info}) # get predictor data_names = ['data', 'im_info'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))] provide_label = [None for i in xrange(len(data))] model_path = cur_path + '/../' + config.output_path + '/' + config.config_name + '/' + config.dataset.image_set + '/' + config.TRAIN.model_prefix arg_params, aux_params = load_param(model_path, config.TEST.test_epoch, process=True) predictor = Predictor(sym, data_names, label_names, context=[mx.gpu(ctx_id[0])], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) # warm up for i in xrange(2): data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0, provide_data=[[ (k, v.shape) for k, v in zip(data_names, data[0]) ]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] _, _, _, _ = im_detect(predictor, data_batch, data_names, scales, config) # 12342_mask Log # LogTxt = open('LogRecorder.txt', 'w') # test for idx, im_name in enumerate(image_names[9001:len(image_names)]): data_batch = mx.io.DataBatch( data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] tic() scores, boxes, masks, data_dict = im_detect(predictor, data_batch, data_names, [1.0], config) im_shapes = [ data_batch.data[i][0].shape[2:4] for i in xrange(len(data_batch.data)) ] # print im_shapes if not config.TEST.USE_MASK_MERGE: all_boxes = [[] for _ in xrange(num_classes)] all_masks = [[] for _ in xrange(num_classes)] nms = py_nms_wrapper(config.TEST.NMS) for j in range(1, num_classes): indexes = np.where(scores[0][:, j] > 0.7)[0] cls_scores = scores[0][indexes, j, np.newaxis] cls_masks = masks[0][indexes, 1, :, :] try: if config.CLASS_AGNOSTIC: cls_boxes = boxes[0][indexes, :] else: raise Exception() except: cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) all_boxes[j] = cls_dets[keep, :] all_masks[j] = cls_masks[keep, :] dets = [all_boxes[j] for j in range(1, num_classes)] masks = [all_masks[j] for j in range(1, num_classes)] else: masks = masks[0][:, 1:, :, :] result_masks, result_dets = gpu_mask_voting( masks, boxes[0], scores[0], num_classes, 100, im_shapes[0][1], im_shapes[0][0], config.TEST.NMS, config.TEST.MASK_MERGE_THRESH, config.BINARY_THRESH, ctx_id[0]) dets = [result_dets[j] for j in range(1, num_classes)] masks = [ result_masks[j][:, 0, :, :] for j in range(1, num_classes) ] print 'testing {} {:.4f}s'.format(im_name, toc()) # visualize for i in xrange(len(dets)): keep = np.where(dets[i][:, -1] > 0.7) dets[i] = dets[i][keep] masks[i] = masks[i][keep] im = cv2.imread(cur_path + '/../' + test_dir + im_name) # im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) ims = show_masks(im, dets, masks, classes, config, 1.0 / scales[0], False) # label = cv2.imread(cur_path + '/../' + label_dir + im_name[0:len(im_name)-4] + '.png') # label = cv2.cvtColor(label, cv2.COLOR_BGR2RGB) # label_flag = checkLabel(label) if dets[0].size: cv2.imwrite( cur_path + '/../' + result_dir + im_name[0:-4] + '_CuBo.png', ims[0]) if dets[1].size: cv2.imwrite( cur_path + '/../' + result_dir + im_name[0:-4] + '_CuFa.png', ims[1]) if dets[2].size: cv2.imwrite( cur_path + '/../' + result_dir + im_name[0:-4] + '_CyBo.png', ims[2]) if dets[3].size: cv2.imwrite( cur_path + '/../' + result_dir + im_name[0:-4] + '_CyFa.png', ims[3]) if dets[4].size: cv2.imwrite( cur_path + '/../' + result_dir + im_name[0:-4] + '_Grip.png', ims[4]) # if np.unique(ims[0]).shape[0] >= 2 and (1 in label_flag): # cv2.imwrite(cur_path + '/../' + result_dir + im_name[0:-4] + '_CuBo.png', ims[0]) # if np.unique(ims[1]).shape[0] >= 2 and (2 in label_flag): # cv2.imwrite(cur_path + '/../' + result_dir + im_name[0:-4] + '_CuFa.png', ims[1]) # if np.unique(ims[2]).shape[0] >= 2 and (3 in label_flag): # cv2.imwrite(cur_path + '/../' + result_dir + im_name[0:-4] + '_CyBo.png', ims[2]) # if np.unique(ims[3]).shape[0] >= 2 and (4 in label_flag): # cv2.imwrite(cur_path + '/../' + result_dir + im_name[0:-4] + '_CyFa.png', ims[3]) # if np.unique(ims[4]).shape[0] >= 2 and (5 in label_flag): # cv2.imwrite(cur_path + '/../' + result_dir + im_name[0:-4] + '_Grip.png', ims[4]) print 'done'
def Seg(self): for i in xrange(2): data_batch = mx.io.DataBatch( data=[self.data], label=[], pad=0, index=0, provide_data=[[(k, v.shape) for k, v in zip(self.data_names, self.data)]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] _, _, _, _ = im_detect(self.predictor, data_batch, self.data_names, scales, config) data_batch = mx.io.DataBatch( data=[self.data], label=[], pad=0, index=0, provide_data=[[(k, v.shape) for k, v in zip(self.data_names, self.data)]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] tic() scores, boxes, masks, data_dict = im_detect(self.predictor, data_batch, self.data_names, scales, config) #print masks #right im_shapes = [ data_batch.data[i][0].shape[2:4] for i in xrange(len(data_batch.data)) ] if not config.TEST.USE_MASK_MERGE: all_boxes = [[] for _ in xrange(self.num_classes)] all_masks = [[] for _ in xrange(self.num_classes)] nms = py_nms_wrapper(config.TEST.NMS) for j in range(1, self.num_classes): indexes = np.where(scores[0][:, j] > 0.7)[0] cls_scores = scores[0][indexes, j, np.newaxis] cls_masks = masks[0][indexes, 1, :, :] try: if config.CLASS_AGNOSTIC: cls_boxes = boxes[0][indexes, :] else: raise Exception() except: cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) all_boxes[j] = cls_dets[keep, :] all_masks[j] = cls_masks[keep, :] dets = [all_boxes[j] for j in range(1, self.num_classes)] masks = [all_masks[j] for j in range(1, self.num_classes)] else: masks = masks[0][:, 1:, :, :] im_height = np.round(im_shapes[0][0] / scales[0]).astype('int') im_width = np.round(im_shapes[0][1] / scales[0]).astype('int') print(im_height, im_width) boxes = clip_boxes(boxes[0], (im_height, im_width)) result_masks, result_dets = gpu_mask_voting( masks, boxes, scores[0], self.num_classes, 100, im_width, im_height, config.TEST.NMS, config.TEST.MASK_MERGE_THRESH, config.BINARY_THRESH, self.ctx_id[0]) dets = [result_dets[j] for j in range(1, self.num_classes)] masks = [ result_masks[j][:, 0, :, :] for j in range(1, self.num_classes) ] for i in xrange(1, len(dets)): keep = np.where(dets[i][:, -1] > 1) dets[i] = dets[i][keep] masks[i] = masks[i][keep] keep = np.where(dets[0][:, -1] > 0.8) dets[0] = dets[0][keep] masks[0] = masks[0][keep] newmask = show_masks(self.fg, dets, masks, self.classes, config) #!!!!!!!! wrong mask self.result = newmask return newmask
def main(): # get symbol ctx_id = [int(i) for i in config.gpus.split(',')] pprint.pprint(config) sym_instance = eval(config.symbol)() sym = sym_instance.get_symbol(config, is_train=False) # set up class names # num_classes = 81 # classes = ['person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', # 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', # 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', # 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', # 'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', # 'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', # 'cake', 'chair', 'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', # 'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', # 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush'] num_classes = 14 #classes = ['screwdriver', 'wrench', 'tomato soup', 'banana', 'jello', 'smallclamp', 'locker', 'mug', 'bigclamp', 'peach', # 'sponge', 'lemon', 'strawberry'] classes = [ 'fg1', 'fg2', 'fg3', 'fg4', 'fg5', 'fg6', 'fg7', 'fg8', 'fg9', 'fg10', 'fg11', 'fg12', 'fg13' ] # load demo data #image_names = ['frame_test.jpg', 'frame0000.jpg', 'frame0199.jpg', 'frame0155.jpg', 'COCO_test2015_000000000275.jpg', 'COCO_test2015_000000001412.jpg', 'COCO_test2015_000000073428.jpg','COCO_test2015_000000393281.jpg'] image_names = ['test_img_bg.jpg'] data = [] for im_name in image_names: assert os.path.exists(cur_path + '/../demo/' + im_name), ( '%s does not exist'.format('../demo/' + im_name)) im = cv2.imread(cur_path + '/../demo/' + im_name, cv2.IMREAD_COLOR) #| cv2.IMREAD_IGNORE_ORIENTATION) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array( [[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data.append({'data': im_tensor, 'im_info': im_info}) # get predictor data_names = ['data', 'im_info'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))] provide_label = [None for i in xrange(len(data))] arg_params, aux_params = load_param(cur_path + '/../model/fcis_coco', 0, process=True) predictor = Predictor(sym, data_names, label_names, context=[mx.gpu(ctx_id[0])], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) # warm up for i in xrange(2): data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0, provide_data=[[ (k, v.shape) for k, v in zip(data_names, data[0]) ]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] _, _, _, _ = im_detect(predictor, data_batch, data_names, scales, config) # test for idx, im_name in enumerate(image_names): data_batch = mx.io.DataBatch( data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] tic() scores, boxes, masks, data_dict = im_detect(predictor, data_batch, data_names, scales, config) im_shapes = [ data_batch.data[i][0].shape[2:4] for i in xrange(len(data_batch.data)) ] if not config.TEST.USE_MASK_MERGE: all_boxes = [[] for _ in xrange(num_classes)] all_masks = [[] for _ in xrange(num_classes)] nms = py_nms_wrapper(config.TEST.NMS) for j in range(1, num_classes): indexes = np.where(scores[0][:, j] > 0.7)[0] cls_scores = scores[0][indexes, j, np.newaxis] cls_masks = masks[0][indexes, 1, :, :] try: if config.CLASS_AGNOSTIC: cls_boxes = boxes[0][indexes, :] else: raise Exception() except: cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) all_boxes[j] = cls_dets[keep, :] all_masks[j] = cls_masks[keep, :] dets = [all_boxes[j] for j in range(1, num_classes)] masks = [all_masks[j] for j in range(1, num_classes)] else: print "test" masks = masks[0][:, 1:, :, :] im_height = np.round(im_shapes[0][0] / scales[0]).astype('int') im_width = np.round(im_shapes[0][1] / scales[0]).astype('int') print(im_height, im_width) boxes = clip_boxes(boxes[0], (im_height, im_width)) result_masks, result_dets = gpu_mask_voting( masks, boxes, scores[0], num_classes, 100, im_width, im_height, config.TEST.NMS, config.TEST.MASK_MERGE_THRESH, config.BINARY_THRESH, ctx_id[0]) dets = [result_dets[j] for j in range(1, num_classes)] masks = [ result_masks[j][:, 0, :, :] for j in range(1, num_classes) ] print 'testing {} {:.4f}s'.format(im_name, toc()) # visualize for i in xrange(len(dets)): keep = np.where(dets[i][:, -1] > 0.1) dets[i] = dets[i][keep] masks[i] = masks[i][keep] im = cv2.imread(cur_path + '/../demo/' + im_name) im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) show_masks(im, dets, masks, classes, config) print 'done'
def predict_on_image_names( image_names, config, model_path_id="/home/data/output/resnet_v1_101_coco_fcis_end2end_ohem-nebraska/train-nebraska/e2e", epoch=8): import argparse import os import sys import logging import pprint import cv2 from utils.image import resize, transform import numpy as np # get config os.environ['PYTHONUNBUFFERED'] = '1' os.environ['MXNET_CUDNN_AUTOTUNE_DEFAULT'] = '0' os.environ['MXNET_ENABLE_GPU_P2P'] = '0' cur_path = os.path.abspath(".") sys.path.insert( 0, os.path.join(cur_path, '../external/mxnet', config.MXNET_VERSION)) import mxnet as mx print("use mxnet at", mx.__file__) from core.tester import im_detect, Predictor from symbols import * from utils.load_model import load_param from utils.show_masks import show_masks from utils.tictoc import tic, toc from nms.nms import py_nms_wrapper from mask.mask_transform import gpu_mask_voting, cpu_mask_voting # get symbol ctx_id = [int(i) for i in config.gpus.split(',')] sym_instance = eval(config.symbol)() sym = sym_instance.get_symbol(config, is_train=False) # set up class names num_classes = 2 classes = ['cp'] # load demo data data = [] for im_name in image_names: assert os.path.exists(im_name), ('%s does not exist'.format(im_name)) im = cv2.imread(im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array( [[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data.append({'data': im_tensor, 'im_info': im_info}) # get predictor data_names = ['data', 'im_info'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))] provide_label = [None for i in xrange(len(data))] # loading the last epoch that was trained, 8 arg_params, aux_params = load_param(model_path_id, epoch, process=True) predictor = Predictor(sym, data_names, label_names, context=[mx.gpu(ctx_id[0])], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) all_classes = [] all_configs = [] all_masks = [] all_dets = [] all_ims = [] # warm up for i in xrange(2): data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0, provide_data=[[ (k, v.shape) for k, v in zip(data_names, data[0]) ]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] _, _, _, _ = im_detect(predictor, data_batch, data_names, scales, config) # test for idx, im_name in enumerate(image_names): data_batch = mx.io.DataBatch( data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] tic() scores, boxes, masks, data_dict = im_detect(predictor, data_batch, data_names, scales, config) im_shapes = [ data_batch.data[i][0].shape[2:4] for i in xrange(len(data_batch.data)) ] if not config.TEST.USE_MASK_MERGE: all_boxes = [[] for _ in xrange(num_classes)] all_masks = [[] for _ in xrange(num_classes)] nms = py_nms_wrapper(config.TEST.NMS) for j in range(1, num_classes): indexes = np.where(scores[0][:, j] > 0.7)[0] cls_scores = scores[0][indexes, j, np.newaxis] cls_masks = masks[0][indexes, 1, :, :] try: if config.CLASS_AGNOSTIC: cls_boxes = boxes[0][indexes, :] else: raise Exception() except: cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) all_boxes[j] = cls_dets[keep, :] all_masks[j] = cls_masks[keep, :] dets = [all_boxes[j] for j in range(1, num_classes)] masks = [all_masks[j] for j in range(1, num_classes)] else: masks = masks[0][:, 1:, :, :] im_height = np.round(im_shapes[0][0] / scales[0]).astype('int') im_width = np.round(im_shapes[0][1] / scales[0]).astype('int') print(im_height, im_width) boxes = clip_boxes(boxes[0], (im_height, im_width)) result_masks, result_dets = gpu_mask_voting( masks, boxes, scores[0], num_classes, 100, im_width, im_height, config.TEST.NMS, config.TEST.MASK_MERGE_THRESH, config.BINARY_THRESH, ctx_id[0]) dets = [result_dets[j] for j in range(1, num_classes)] masks = [ result_masks[j][:, 0, :, :] for j in range(1, num_classes) ] print('testing {} {:.4f}s'.format(im_name, toc())) # visualize for i in xrange(len(dets)): keep = np.where(dets[i][:, -1] > 0.7) dets[i] = dets[i][keep] masks[i] = masks[i][keep] all_classes.append(classes) all_configs.append(config) all_masks.append(masks) all_dets.append(dets) im = cv2.imread(im_name) all_ims.append(im) return all_ims, all_dets, all_masks, all_configs, all_classes
def main(): parser = argparse.ArgumentParser() parser.add_argument("indir", type=lambda s: unicode(s, 'utf8'), help="Directory containing list of images") parser.add_argument("outfile", type=lambda s: unicode(s, 'utf8'), help="Path to write predictions") parser.add_argument("-d", "--device", type=int, default=0, help="Device ID to use") args = parser.parse_args() params = vars(args) # ---------------------------------------------------------- Read config ctx_id = [int(i) for i in config.gpus.split(',')] pprint.pprint(config) sym_instance = eval(config.symbol)() sym = sym_instance.get_symbol(config, is_train=False) # set up class names num_classes = 81 classes = [ 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush' ] config['gpus'] = str(params['device']) # ---------------------------------------------------------- Load Images image_path_list = [] data = [] scale_factor = 1.0 img_dir = osp.abspath(params['indir']) det_thresh = 0.7 # Load abs paths of images for f in sorted(os.listdir(img_dir)): _, f_ext = osp.splitext(f) if f_ext in ['.jpg', '.png', '.jpeg']: f_path = osp.join(img_dir, f) image_path_list.append(f_path) print 'Loading {} images into memory...'.format(len(image_path_list)) for image_path in image_path_list: im = cv2.imread(image_path, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) height, width = im.shape[:2] im = cv2.resize( im, (int(scale_factor * width), int(scale_factor * height))) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array( [[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data.append({'data': im_tensor, 'im_info': im_info}) print 'Loaded {} images'.format(len(image_path_list)) # ---------------------------------------------------------- Predict predictions = [] # get predictor data_names = ['data', 'im_info'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))] provide_label = [None for i in xrange(len(data))] arg_params, aux_params = load_param( '/BS/orekondy2/work/opt/FCIS/model/fcis_coco', 0, process=True) predictor = Predictor(sym, data_names, label_names, context=[mx.gpu(ctx_id[0])], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) # warm up for i in xrange(2): data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0, provide_data=[[ (k, v.shape) for k, v in zip(data_names, data[0]) ]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] _, _, _, _ = im_detect(predictor, data_batch, data_names, scales, config) # test for idx, image_path in enumerate(image_path_list): data_batch = mx.io.DataBatch( data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] tic() scores, boxes, masks, data_dict = im_detect(predictor, data_batch, data_names, scales, config) im_shapes = [ data_batch.data[i][0].shape[2:4] for i in xrange(len(data_batch.data)) ] if not config.TEST.USE_MASK_MERGE: all_boxes = [[] for _ in xrange(num_classes)] all_masks = [[] for _ in xrange(num_classes)] nms = py_nms_wrapper(config.TEST.NMS) for j in range(1, num_classes): indexes = np.where(scores[0][:, j] > 0.7)[0] cls_scores = scores[0][indexes, j, np.newaxis] cls_masks = masks[0][indexes, 1, :, :] try: if config.CLASS_AGNOSTIC: cls_boxes = boxes[0][indexes, :] else: raise Exception() except: cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) all_boxes[j] = cls_dets[keep, :] all_masks[j] = cls_masks[keep, :] dets = [all_boxes[j] for j in range(1, num_classes)] masks = [all_masks[j] for j in range(1, num_classes)] else: masks = masks[0][:, 1:, :, :] im_height = np.round(im_shapes[0][0] / scales[0]).astype('int') im_width = np.round(im_shapes[0][1] / scales[0]).astype('int') print(im_height, im_width) boxes = clip_boxes(boxes[0], (im_height, im_width)) result_masks, result_dets = gpu_mask_voting( masks, boxes, scores[0], num_classes, 100, im_width, im_height, config.TEST.NMS, config.TEST.MASK_MERGE_THRESH, config.BINARY_THRESH, ctx_id[0]) dets = [result_dets[j] for j in range(1, num_classes)] masks = [ result_masks[j][:, 0, :, :] for j in range(1, num_classes) ] print '{} testing {} {:.4f}s'.format(idx, image_path, toc()) # visualize for i in xrange(len(dets)): keep = np.where(dets[i][:, -1] > det_thresh) dets[i] = dets[i][keep] masks[i] = masks[i][keep] im = cv2.imread(image_path_list[idx]) im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) org_height, org_width = cv2.imread(image_path_list[idx]).shape[:2] # im = cv2.resize(im,(int(scale_factor*org_width), int(scale_factor*org_height))) """ visualize all detections in one image :param im_array: [b=1 c h w] in rgb :param detections: [ numpy.ndarray([[x1 y1 x2 y2 score]]) for j in classes ] :param class_names: list of names in imdb :param scale: visualize the scaled image :return: """ detections = dets class_names = classes cfg = config scale = 1.0 person_idx = class_names.index('person') dets = detections[person_idx] msks = masks[person_idx] for mask_idx, (det, msk) in enumerate(zip(dets, msks)): inst_arr = np.zeros_like(im[:, :, 0]) # Create a 2D W x H array bbox = det[:4] * scale cod = bbox.astype(int) if im[cod[1]:cod[3], cod[0]:cod[2], 0].size > 0: msk = cv2.resize( msk, im[cod[1]:cod[3] + 1, cod[0]:cod[2] + 1, 0].T.shape) bimsk = (msk >= cfg.BINARY_THRESH).astype('uint8') # ------- Create bit-mask for this instance inst_arr[cod[1]:cod[3] + 1, cod[0]:cod[2] + 1] = bimsk # Add thresholded binary mask rs_inst_arr = scipy.misc.imresize(inst_arr, (org_height, org_width)) rle = mask.encode(np.asfortranarray(rs_inst_arr)) predictions.append({ 'image_path': image_path, 'label': 'person', 'segmentation': rle, 'bbox': bbox.tolist(), 'score': det[-1], }) del msk del bimsk del rs_inst_arr print 'Created {} predictions'.format(len(predictions)) # ---------------------------------------------------------- Write output with open(params['outfile'], 'wb') as wf: json.dump(predictions, wf, indent=2)