import numpy as np
import skimage.io as io
import matplotlib.pyplot as plt
import pylab
from PIL import Image
pylab.rcParams['figure.figsize'] = (8.0, 10.0)
print('Start')
dataDir = '..'
dataType = 'val2014'
annFile = '{}/annotations/captions_{}.json'.format(dataDir, dataType)
print(annFile)
coco_caps = COCO(annFile)
IMG_ID = 115006
# catIds = coco_caps.getCatIds(catNms=['person','dog','skateboard']);
# imgIds = coco_caps.getImgIds(catIds=catIds );
imgIds = coco_caps.getImgIds(imgIds=[IMG_ID])
img = coco_caps.loadImgs(imgIds[np.random.randint(0, len(imgIds))])[0]
img_name = img['file_name']
img_path = '/home/wsh/coco/val2014/' + img_name
image = Image.open(img_path)
res = None
annIds = coco_caps.getAnnIds(imgIds=img['id'])
anns = coco_caps.loadAnns(annIds)
print('---------------------------', IMG_ID,
from os.path import isdir, isfile, join, basename as bn
import json
import numpy as np
import cv2
import matplotlib.pyplot as plt
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
from image import gaussian_radius, draw_gaussian
annType = 'bbox'
gt_fn = '../data/coco/annotations/instances_test2020.json'
cocoGt = COCO(gt_fn)
dt_fn = '../ckpt/results_sl1_catspec_e70.json'
cocoDt = cocoGt.loadRes(dt_fn)
# running evaluation
cocoEval = COCOeval(cocoGt,cocoDt,annType)
leparams = cocoEval.params
leparams.iouThrs = [0.2,]
leparams.maxDets = [100,]
leparams.areaRng = [[0 ** 2, 1e5 ** 2],]
leparams.areaRngLbl = ['all',]
cocoEval.evaluate()
with open(gt_fn) as f:
gt_coco = json.load(f)
def test(cfg,
data,
weights=None,
batch_size=16,
img_size=416,
conf_thres=0.001,
iou_thres=0.6, # for nms
save_json=False,
single_cls=False,
augment=False,
model=None,
data_loader=None):
# Initialize/load model and set device
if model is None:
device = torch_utils.select_device(opt.device, batch_size=batch_size)
verbose = opt.task == 'test'
# Remove previous
for f in glob.glob('test_batch*.jpg'):
os.remove(f)
# Initialize model
model = Darknet(cfg, img_size)
# Load weights
attempt_download(weights)
if weights.endswith('.pt'): # pytorch format
model.load_state_dict(torch.load(weights, map_location=device)['model'])
else: # darknet format
load_darknet_weights(model, weights)
# Fuse
model.fuse()
model.to(device)
model.half() # fp16?
if device.type != 'cpu' and torch.cuda.device_count() > 1:
model = nn.DataParallel(model)
else: # called by train.py
device = next(model.parameters()).device # get model device
verbose = False
# Configure run
data = parse_data_cfg(data)
nc = 1 if single_cls else int(data['classes']) # number of classes
path = data['valid'] # path to test images
names = load_classes(data['names']) # class names
iouv = torch.linspace(0.5, 0.95, 10).to(device) # iou vector for [email protected]:0.95
iouv = iouv[0].view(1) # comment for [email protected]:0.95
niou = iouv.numel()
# Data loader
if data_loader is None:
dataset = LoadImagesAndLabels(path, img_size, batch_size, rect=True, single_cls=opt.single_cls)
batch_size = min(batch_size, len(dataset))
data_loader = DataLoader(dataset,
batch_size=batch_size,
num_workers=min([os.cpu_count(), batch_size if batch_size > 1 else 0, 8]),
pin_memory=True,
collate_fn=dataset.collate_fn)
seen = 0
model.eval()
_ = model(
torch.zeros((1, 3, img_size, img_size), device=device).half()) if device.type != 'cpu' else None # run once
coco91class = coco80_to_coco91_class()
s = ('%20s' + '%10s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R', '[email protected]', 'F1')
p, r, f1, mp, mr, map, mf1, t0, t1 = 0., 0., 0., 0., 0., 0., 0., 0., 0.
loss = torch.zeros(3, device=device)
jdict, stats, ap, ap_class = [], [], [], []
for batch_i, (imgs, targets, paths, shapes) in enumerate(tqdm(data_loader, desc=s)):
imgs = imgs.to(device).half() / 255.0 # uint8 to float32, 0 - 255 to 0.0 - 1.0
targets = targets.to(device)
nb, _, height, width = imgs.shape # batch size, channels, height, width
whwh = torch.Tensor([width, height, width, height]).to(device)
# Plot images with bounding boxes
f = 'test_batch%g.jpg' % batch_i # filename
# if batch_i < 1 and not os.path.exists(f):
# plot_images(imgs=imgs, targets=targets, paths=paths, fname=f)
# Disable gradients
with torch.no_grad():
# Run model
t = torch_utils.time_synchronized()
inf_out, train_out = model(imgs, augment=augment) # inference and training outputs
t0 += torch_utils.time_synchronized() - t
# Compute loss
if hasattr(model, 'hyp'): # if model has loss hyperparameters
loss += compute_loss(train_out, targets, model)[1][:3] # GIoU, obj, cls
# Run NMS
t = torch_utils.time_synchronized()
output = non_max_suppression(inf_out, conf_thres=conf_thres, iou_thres=iou_thres) # nms
t1 += torch_utils.time_synchronized() - t
# Statistics per image
for si, pred in enumerate(output):
labels = targets[targets[:, 0] == si, 1:]
nl = len(labels)
tcls = labels[:, 0].tolist() if nl else [] # target class
seen += 1
if pred is None:
if nl:
stats.append((torch.zeros(0, niou, dtype=torch.bool), torch.Tensor(), torch.Tensor(), tcls))
continue
# Append to text file
# with open('test.txt', 'a') as file:
# [file.write('%11.5g' * 7 % tuple(x) + '\n') for x in pred]
# Clip boxes to image bounds
clip_coords(pred, (height, width))
# Append to pycocotools JSON dictionary
if save_json:
# [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ...
image_id = int(Path(paths[si]).stem.split('_')[-1])
box = pred[:, :4].clone() # xyxy
scale_coords(imgs[si].shape[1:], box, shapes[si][0], shapes[si][1]) # to original shape
box = xyxy2xywh(box) # xywh
box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner
for p, b in zip(pred.tolist(), box.tolist()):
jdict.append({'image_id': image_id,
'category_id': coco91class[int(p[5])],
'bbox': [round(x, 3) for x in b],
'score': round(p[4], 5)})
# Assign all predictions as incorrect
correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool, device=device)
if nl:
detected = [] # target indices
tcls_tensor = labels[:, 0]
# target boxes
tbox = xywh2xyxy(labels[:, 1:5]) * whwh
# Per target class
for cls in torch.unique(tcls_tensor):
ti = (cls == tcls_tensor).nonzero().view(-1) # prediction indices
pi = (cls == pred[:, 5]).nonzero().view(-1) # target indices
# Search for detections
if pi.shape[0]:
# Prediction to target ious
ious, i = box_iou(pred[pi, :4], tbox[ti]).max(1) # best ious, indices
# Append detections
for j in (ious > iouv[0]).nonzero():
d = ti[i[j]] # detected target
if d not in detected:
detected.append(d)
correct[pi[j]] = ious[j] > iouv # iou_thres is 1xn
if len(detected) == nl: # all targets already located in image
break
# Append statistics (correct, conf, pcls, tcls)
stats.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls))
# Compute statistics
stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy
if len(stats):
p, r, ap, f1, ap_class = ap_per_class(*stats)
if niou > 1:
p, r, ap, f1 = p[:, 0], r[:, 0], ap.mean(1), ap[:, 0] # [P, R, [email protected]:0.95, [email protected]]
mp, mr, map, mf1 = p.mean(), r.mean(), ap.mean(), f1.mean()
nt = np.bincount(stats[3].astype(np.int64), minlength=nc) # number of targets per class
else:
nt = torch.zeros(1)
# Print results
pf = '%20s' + '%10.3g' * 6 # print format
print(pf % ('all', seen, nt.sum(), mp, mr, map, mf1))
# Print results per class
if verbose and nc > 1 and len(stats):
for i, c in enumerate(ap_class):
print(pf % (names[c], seen, nt[c], p[i], r[i], ap[i], f1[i]))
# Print speeds
if verbose or save_json:
t = tuple(x / seen * 1E3 for x in (t0, t1, t0 + t1)) + (img_size, img_size, batch_size) # tuple
print('Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g' % t)
maps = np.zeros(nc) + map
# Save JSON
if save_json and map and len(jdict):
print('\nCOCO mAP with pycocotools...')
imgIds = [int(Path(x).stem.split('_')[-1]) for x in data_loader.dataset.img_files]
with open('results.json', 'w') as file:
json.dump(jdict, file)
try:
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
except:
print('WARNING: missing pycocotools package, can not compute official COCO mAP. See requirements.txt.')
# https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb
cocoGt = COCO(glob.glob('../coco/annotations/instances_val*.json')[0]) # initialize COCO ground truth api
cocoDt = cocoGt.loadRes('results.json') # initialize COCO pred api
cocoEval = COCOeval(cocoGt, cocoDt, 'bbox')
cocoEval.params.imgIds = imgIds # [:32] # only evaluate these images
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
map, map50 = cocoEval.stats[:2] # update results ([email protected]:0.95, [email protected])
return (mp, mr, map50, map, *(loss.cpu() / len(data_loader)).tolist()), maps, t
# Return results
for i, c in enumerate(ap_class):
maps[c] = ap[i]
return (mp, mr, map, mf1, *(loss.cpu() / len(data_loader)).tolist()), maps
from pycocotools.coco import COCO
from numpy import *
import os
data_dir = '/mnt/fcav/self_training/object_detection/dataset/cityscapes/annotations/instances_caronly_val.json'
output_caronly_dir = '/mnt/fcav/self_training/object_detection/dataset/cityscapes/annotations_txt'
if not os.path.exists(output_caronly_dir):
os.makedirs(output_caronly_dir)
# initialize COCO api for instance annotations
coco = COCO(data_dir)
imgs = coco.loadImgs(list(coco.imgs.keys()))
for k, img in enumerate(imgs):
print('[%6d/%6d] %s' % (k, len(imgs), img['file_name']))
anns = coco.loadAnns(coco.getAnnIds(imgIds=img['id']))
image_name, image_ext = os.path.splitext(img['file_name'])
result_file_car_only = open(
os.path.join(output_caronly_dir,
img['file_name'].replace(image_ext, '.txt')), 'w')
for ann in anns:
category = ann['category_id']
bbox = ann['bbox']
label = 'Car'
result_file_car_only.write(
"%s -1 -1 -10 %.3f %.3f %.3f %.3f -1 -1 -1 -1000 -1000 -1000 -10\n"
% (label, bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3]))
def evaluate(model: tf.keras.Model,
dataset: tf.data.Dataset,
class2idx: Mapping[str, int],
steps: int,
print_every: int = 10):
gt_coco = dict(images=[], annotations=[])
results_coco = []
image_id = 1
annot_id = 1
# Create COCO categories
categories = [
dict(supercategory='instance', id=i, name=n)
for n, i in class2idx.items()
]
gt_coco['categories'] = categories
for i, (images, (labels, bbs)) in enumerate(dataset):
bboxes, categories, scores = model(images, training=False)
h, w = images.shape[1:3]
# Iterate through images in batch, and for each one
# create the ground truth coco annotation
for batch_idx in range(len(bboxes)):
gt_labels, gt_boxes = labels[batch_idx], bbs[batch_idx]
no_padding_mask = gt_labels != -1
gt_labels = tf.boolean_mask(gt_labels, no_padding_mask)
gt_boxes = tf.boolean_mask(gt_boxes, no_padding_mask)
im_annot, annots = _COCO_gt_annot(image_id, annot_id, (h, w),
gt_labels, gt_boxes)
gt_coco['annotations'].extend(annots)
gt_coco['images'].append(im_annot)
preds = categories[batch_idx], bboxes[batch_idx], scores[batch_idx]
pred_labels, pred_boxes, pred_scores = preds
if pred_labels.shape[0] > 0:
results = _COCO_result(image_id, pred_labels, pred_boxes,
pred_scores)
results_coco.extend(results)
annot_id += len(annots)
image_id += 1
if i % print_every == 0:
print(f'Validating[{i}/{steps}]...')
# Convert custom annotations to COCO annots
gtCOCO = COCO()
gtCOCO.dataset = gt_coco
gtCOCO.createIndex()
resCOCO = COCO()
resCOCO.dataset['images'] = [img for img in gt_coco['images']]
resCOCO.dataset['categories'] = copy.deepcopy(gt_coco['categories'])
for i, ann in enumerate(results_coco):
bb = ann['bbox']
x1, x2, y1, y2 = [bb[0], bb[0] + bb[2], bb[1], bb[1] + bb[3]]
if not 'segmentation' in ann:
ann['segmentation'] = [[x1, y1, x1, y2, x2, y2, x2, y1]]
ann['area'] = bb[2] * bb[3]
ann['id'] = i + 1
ann['iscrowd'] = 0
resCOCO.dataset['annotations'] = results_coco
resCOCO.createIndex()
coco_eval = COCOeval(gtCOCO, resCOCO, 'bbox')
coco_eval.params.imgIds = sorted(gtCOCO.getImgIds())
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
from pycocotools.coco import COCO import numpy as np import skimage.io as io import matplotlib.pyplot as plt # In[2]: dataDir = '..' dataType = 'val2014' annFile = '%s/annotations/instances_%s.json' % (dataDir, dataType) # In[3]: # initialize COCO api for instance annotations coco = COCO(annFile) # In[4]: # display COCO categories and supercategories cats = coco.loadCats(coco.getCatIds()) nms = [cat['name'] for cat in cats] print 'COCO categories: \n', ' '.join(nms) print '\n' nms = set([cat['supercategory'] for cat in cats]) print 'COCO supercategories: \n', ' '.join(nms) # In[5]: # get all images containing given categories, select one at random catIds = coco.getCatIds(catNms=['person', 'dog', 'skateboard'])
def load_coco_dection_dataset(imgs_dir,
annotations_filepath,
shuffle_img=True):
"""Load data from dataset by pycocotools. This tools can be download from "http://mscoco.org/dataset/#download"
Args:
imgs_dir: directories of coco images
annotations_filepath: file path of coco annotations file
shuffle_img: wheter to shuffle images order
Return:
coco_data: list of dictionary format information of each image
"""
coco = COCO(annotations_filepath)
img_ids = coco.getImgIds() # totally 82783 images
cat_ids = coco.getCatIds(
) # totally 90 catagories, however, the number of categories is not continuous, \
# [0,12,26,29,30,45,66,68,69,71,83] are missing, this is the problem of coco dataset.
if shuffle_img:
shuffle(img_ids)
coco_data = []
nb_imgs = len(img_ids)
for index, img_id in enumerate(img_ids):
if index % 100 == 0:
print("Readling images: %d / %d " % (index, nb_imgs))
img_info = {}
bboxes = []
labels = []
img_detail = coco.loadImgs(img_id)[0]
pic_height = img_detail['height']
pic_width = img_detail['width']
ann_ids = coco.getAnnIds(imgIds=img_id, catIds=cat_ids)
anns = coco.loadAnns(ann_ids)
LABELS_NEED = {1, 2, 3, 4, 6, 7, 8, 10, 12, 13}
for ann in anns:
category_id = ann["category_id"]
if not category_id in LABELS_NEED:
continue
bboxes_data = ann['bbox']
bboxes_data = [bboxes_data[0]/float(pic_width), bboxes_data[1]/float(pic_height),\
bboxes_data[2]/float(pic_width), bboxes_data[3]/float(pic_height)]
# the format of coco bounding boxs is [Xmin, Ymin, width, height]
bboxes.append(bboxes_data)
#labels.append(ann['category_id'])
if category_id in range(1, 5):
labels.append(category_id)
elif category_id == 6:
labels.append(5)
elif category_id == 7:
labels.append(6)
elif category_id == 8:
labels.append(7)
elif category_id == 10:
labels.append(8)
elif category_id == 12:
labels.append(9)
elif category_id == 13:
labels.append(10)
if len(labels) == 0:
continue
img_path = os.path.join(imgs_dir, img_detail['file_name'])
img_bytes = tf.gfile.FastGFile(img_path, 'rb').read()
img_info['pixel_data'] = img_bytes
img_info['height'] = pic_height
img_info['width'] = pic_width
img_info['bboxes'] = bboxes
img_info['labels'] = labels
coco_data.append(img_info)
return coco_data
parser.add_argument('--gt_ann_file',
default='data/coco/annotations/instances_val2017.json',
type=str)
parser.add_argument('--eval_type',
default='both',
choices=['bbox', 'mask', 'both'],
type=str)
args = parser.parse_args()
if __name__ == '__main__':
eval_bbox = (args.eval_type in ('bbox', 'both'))
eval_mask = (args.eval_type in ('mask', 'both'))
print('Loading annotations..')
gt_annotations = COCO(args.gt_ann_file)
if eval_bbox:
bbox_dets = gt_annotations.loadRes(args.bbox_det_file)
if eval_mask:
mask_dets = gt_annotations.loadRes(args.mask_det_file)
if eval_bbox:
print('\nEvaluating BBoxes:')
bbox_eval = COCOeval(gt_annotations, bbox_dets, 'bbox')
bbox_eval.evaluate()
bbox_eval.accumulate()
bbox_eval.summarize()
if eval_mask:
print('\nEvaluating Masks:')
bbox_eval = COCOeval(gt_annotations, mask_dets, 'segm')
def test(cfg,
data_cfg,
weights=None,
batch_size=16,
img_size=416,
iou_thres=0.5,
conf_thres=0.001,
nms_thres=0.5,
save_json=False,
model=None):
if model is None:
device = torch_utils.select_device()
# Initialize model
model = Darknet(cfg, img_size).to(device)
# Load weights
if weights.endswith('.pt'): # pytorch format
model.load_state_dict(
torch.load(weights, map_location=device)['model'])
else: # darknet format
_ = load_darknet_weights(model, weights)
if torch.cuda.device_count() > 1:
model = nn.DataParallel(model)
else:
device = next(model.parameters()).device # get model device
# Configure run
data_cfg = parse_data_cfg(data_cfg)
nc = int(data_cfg['classes']) # number of classes
test_path = data_cfg['valid'] # path to test images
names = load_classes(data_cfg['names']) # class names
# Dataloader
dataset = LoadImagesAndLabels(test_path, img_size, batch_size)
dataloader = DataLoader(dataset,
batch_size=batch_size,
num_workers=4,
pin_memory=True,
collate_fn=dataset.collate_fn)
seen = 0
model.eval()
coco91class = coco80_to_coco91_class()
print(('%20s' + '%10s' * 6) %
('Class', 'Images', 'Targets', 'P', 'R', 'mAP', 'F1'))
loss, p, r, f1, mp, mr, map, mf1 = 0., 0., 0., 0., 0., 0., 0., 0.
jdict, stats, ap, ap_class = [], [], [], []
for batch_i, (imgs, targets, paths,
shapes) in enumerate(tqdm(dataloader, desc='Computing mAP')):
targets = targets.to(device)
imgs = imgs.to(device)
_, _, height, width = imgs.shape # batch size, channels, height, width
# Plot images with bounding boxes
if batch_i == 0 and not os.path.exists('test_batch0.jpg'):
plot_images(imgs=imgs, targets=targets, fname='test_batch0.jpg')
# Run model
inf_out, train_out = model(imgs) # inference and training outputs
# Compute loss
if hasattr(model, 'hyp'): # if model has loss hyperparameters
loss_i, _ = compute_loss(train_out, targets, model)
loss += loss_i.item()
# Run NMS
output = non_max_suppression(inf_out,
conf_thres=conf_thres,
nms_thres=nms_thres)
# Statistics per image
for si, pred in enumerate(output):
labels = targets[targets[:, 0] == si, 1:]
nl = len(labels)
tcls = labels[:, 0].tolist() if nl else [] # target class
seen += 1
if pred is None:
if nl:
stats.append(([], torch.Tensor(), torch.Tensor(), tcls))
continue
# Append to text file
# with open('test.txt', 'a') as file:
# [file.write('%11.5g' * 7 % tuple(x) + '\n') for x in pred]
# Append to pycocotools JSON dictionary
if save_json:
# [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ...
image_id = int(Path(paths[si]).stem.split('_')[-1])
box = pred[:, :4].clone() # xyxy
scale_coords(imgs[si].shape[1:], box,
shapes[si]) # to original shape
box = xyxy2xywh(box) # xywh
box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner
for di, d in enumerate(pred):
jdict.append({
'image_id': image_id,
'category_id': coco91class[int(d[6])],
'bbox': [float3(x) for x in box[di]],
'score': float(d[4])
})
# Assign all predictions as incorrect
correct = [0] * len(pred)
if nl:
detected = []
tcls_tensor = labels[:, 0]
# target boxes
tbox = xywh2xyxy(labels[:, 1:5])
tbox[:, [0, 2]] *= width
tbox[:, [1, 3]] *= height
# Search for correct predictions
for i, (*pbox, pconf, pcls_conf, pcls) in enumerate(pred):
# Break if all targets already located in image
if len(detected) == nl:
break
# Continue if predicted class not among image classes
if pcls.item() not in tcls:
continue
# Best iou, index between pred and targets
m = (pcls == tcls_tensor).nonzero().view(-1)
iou, bi = bbox_iou(pbox, tbox[m]).max(0)
# If iou > threshold and class is correct mark as correct
if iou > iou_thres and m[
bi] not in detected: # and pcls == tcls[bi]:
correct[i] = 1
detected.append(m[bi])
# Append statistics (correct, conf, pcls, tcls)
stats.append((correct, pred[:, 4].cpu(), pred[:, 6].cpu(), tcls))
# Compute statistics
stats = [np.concatenate(x, 0) for x in list(zip(*stats))] # to numpy
nt = np.bincount(stats[3].astype(np.int64),
minlength=nc) # number of targets per class
if len(stats):
p, r, ap, f1, ap_class = ap_per_class(*stats)
mp, mr, map, mf1 = p.mean(), r.mean(), ap.mean(), f1.mean()
# Print results
pf = '%20s' + '%10.3g' * 6 # print format
print(pf % ('all', seen, nt.sum(), mp, mr, map, mf1))
# Print results per class
if nc > 1 and len(stats):
for i, c in enumerate(ap_class):
print(pf % (names[c], seen, nt[c], p[i], r[i], ap[i], f1[i]))
# Save JSON
if save_json and map and len(jdict):
imgIds = [int(Path(x).stem.split('_')[-1]) for x in dataset.img_files]
with open('results.json', 'w') as file:
json.dump(jdict, file)
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
# https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb
cocoGt = COCO('../coco/annotations/instances_val2014.json'
) # initialize COCO ground truth api
cocoDt = cocoGt.loadRes('results.json') # initialize COCO pred api
cocoEval = COCOeval(cocoGt, cocoDt, 'bbox')
cocoEval.params.imgIds = imgIds # [:32] # only evaluate these images
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
map = cocoEval.stats[1] # update mAP to pycocotools mAP
# Return results
maps = np.zeros(nc) + map
for i, c in enumerate(ap_class):
maps[c] = ap[i]
return (mp, mr, map, mf1, loss / len(dataloader)), maps
def main():
if len(sys.argv) != 6:
raise ValueError(
"Please specify args: $> python run_analysis.py [annotations_path] [detections_path] [save_dir] [team_name] [version_name]"
)
latex_jinja_env = jinja2.Environment(block_start_string='\BLOCK{',
block_end_string='}',
variable_start_string='\VAR{',
variable_end_string='}',
comment_start_string='\#{',
comment_end_string='}',
line_statement_prefix='%%',
line_comment_prefix='%#',
trim_blocks=True,
autoescape=False,
loader=jinja2.FileSystemLoader(
os.path.abspath('./latex/')))
template = latex_jinja_env.get_template('report_template.tex')
template_vars = {}
annFile = sys.argv[1]
splitName = annFile.split("/")[-1].replace("_", "\\_")
resFile = sys.argv[2]
print("{:10}[{}]\n{:10}[{}]".format('annFile:', annFile, 'resFile:',
resFile))
saveDir = sys.argv[3]
if not os.path.exists(saveDir):
os.makedirs(saveDir)
teamName = sys.argv[4]
versionName = sys.argv[5]
## create dictionary with all images info
gt_data = json.load(open(annFile, 'rb'))
imgs_info = {
i['id']: {
'id': i['id'],
'width': i['width'],
'height': i['height'],
'coco_url': i['coco_url']
}
for i in gt_data['images']
}
## load team detections
dt_data = json.load(open(resFile, 'rb'))
team_dts = {}
for d in dt_data:
if d['image_id'] in team_dts: team_dts[d['image_id']].append(d)
else: team_dts[d['image_id']] = [d]
team_split_dts = []
for img_id in team_dts:
if img_id in imgs_info:
team_split_dts.extend(
sorted(team_dts[img_id], key=lambda k: -k['score'])[:20])
print("Loaded [{}] detections from [{}] images.".format(
len(team_split_dts), len(imgs_info)))
template_vars['team_name'] = teamName
template_vars['version_name'] = versionName
template_vars['split_name'] = splitName
template_vars['num_dts'] = len(team_split_dts)
template_vars['num_imgs_dts'] = len(
set([d['image_id'] for d in team_split_dts]))
template_vars['num_imgs'] = len(imgs_info)
## load ground truth annotations
coco_gt = COCO(annFile)
## initialize COCO detections api
coco_dt = coco_gt.loadRes(team_split_dts)
## initialize COCO analyze api
coco_analyze = COCOanalyze(coco_gt, coco_dt, 'keypoints')
## initialize list of image IDs
imgIds_file = open('./coco-minival500_images.txt')
imgIds_str = imgIds_file.readline()
if imgIds_str[-1] == '\n':
imgIds_str = imgIds_str[:-1]
imgIds_str = imgIds_str.split(',')
imgIds = []
for x in imgIds_str:
imgIds.append(int(x))
print("The length of list is: %d" % (len(imgIds)))
coco_analyze.cocoEval.params.imgIds = imgIds
## regular evaluation
coco_analyze.evaluate(verbose=True,
makeplots=True,
savedir=saveDir,
team_name=teamName)
template_vars['overall_prc_medium'] = '%s/prc_[%s][medium][%d].pdf' % (
saveDir, teamName, coco_analyze.params.maxDets[0])
template_vars['overall_prc_large'] = '%s/prc_[%s][large][%d].pdf' % (
saveDir, teamName, coco_analyze.params.maxDets[0])
template_vars['overall_prc_all'] = '%s/prc_[%s][all][%d].pdf' % (
saveDir, teamName, coco_analyze.params.maxDets[0])
############################################################################
# COMMENT OUT ANY OF THE BELOW TO SKIP FROM ANALYSIS
## analyze imapct on AP of all error types
paths = errorsAPImpact(coco_analyze, saveDir)
template_vars.update(paths)
## analyze breakdown of localization errors
paths = localizationErrors(coco_analyze, imgs_info, saveDir)
template_vars.update(paths)
## analyze scoring errors
paths = scoringErrors(coco_analyze, .75, imgs_info, saveDir)
template_vars.update(paths)
## analyze background false positives
paths = backgroundFalsePosErrors(coco_analyze, imgs_info, saveDir)
template_vars.update(paths)
## analyze background false negatives
paths = backgroundFalseNegErrors(coco_analyze, imgs_info, saveDir)
template_vars.update(paths)
## analyze sensitivity to occlusion and crowding of instances
paths = occlusionAndCrowdingSensitivity(coco_analyze, .75, saveDir)
template_vars.update(paths)
## analyze sensitivity to size of instances
paths = sizeSensitivity(coco_analyze, .75, saveDir)
template_vars.update(paths)
output_report = open('./%s_performance_report.tex' % teamName, 'w')
output_report.write(template.render(template_vars))
output_report.close()
def test(
data,
weights=None,
batch_size=16,
imgsz=640,
conf_thres=0.001,
iou_thres=0.6, # for NMS
save_json=False,
single_cls=False,
augment=False,
verbose=False,
model=None,
dataloader=None,
save_dir='',
merge=False,
save_txt=False,
):
# Initialize/load model and set device
training = model is not None
if training: # called by train.py
device = next(model.parameters()).device # get model device
else: # called directly
set_logging()
device = select_device(opt.device, batch_size=batch_size)
merge, save_txt = opt.merge, opt.save_txt # use Merge NMS, save *.txt labels
if save_txt:
out = Path('inference/output')
if os.path.exists(out):
shutil.rmtree(out) # delete output folder
os.makedirs(out) # make new output folder
# Remove previous
for f in glob.glob(str(Path(save_dir) / 'test_batch*.jpg')):
os.remove(f)
# Load model
model = attempt_load(weights, map_location=device) # load FP32 model
imgsz = check_img_size(imgsz, s=model.stride.max()) # check img_size
# Multi-GPU disabled, incompatible with .half() https://github.com/ultralytics/yolov5/issues/99
# if device.type != 'cpu' and torch.cuda.device_count() > 1:
# model = nn.DataParallel(model)
# Half
half = device.type != 'cpu' # half precision only supported on CUDA
if half:
model.half()
# Configure
model.eval()
with open(data) as f:
data = yaml.load(f, Loader=yaml.FullLoader) # model dict
check_dataset(data) # check
nc = 1 if single_cls else int(data['nc']) # number of classes
iouv = torch.linspace(0.5, 0.95,
10).to(device) # iou vector for [email protected]:0.95
niou = iouv.numel()
# Dataloader
if not training:
img = torch.zeros((1, 3, imgsz, imgsz), device=device) # init img
_ = model(img.half() if half else img
) if device.type != 'cpu' else None # run once
path = data['test'] if opt.task == 'test' else data[
'val'] # path to val/test images
dataloader = create_dataloader(path,
imgsz,
batch_size,
model.stride.max(),
opt,
hyp=None,
augment=False,
cache=False,
pad=0.5,
rect=True)[0]
seen = 0
names = model.names if hasattr(model, 'names') else model.module.names
coco91class = coco80_to_coco91_class()
s = ('%20s' + '%12s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R',
'[email protected]', '[email protected]:.95')
p, r, f1, mp, mr, map50, map, t0, t1 = 0., 0., 0., 0., 0., 0., 0., 0., 0.
loss = torch.zeros(3, device=device)
jdict, stats, ap, ap_class = [], [], [], []
for batch_i, (img, targets, paths,
shapes) in enumerate(tqdm(dataloader, desc=s)):
img = img.to(device, non_blocking=True)
img = img.half() if half else img.float() # uint8 to fp16/32
img /= 255.0 # 0 - 255 to 0.0 - 1.0
targets = targets.to(device)
nb, _, height, width = img.shape # batch size, channels, height, width
whwh = torch.Tensor([width, height, width, height]).to(device)
# Disable gradients
with torch.no_grad():
# Run model
t = time_synchronized()
inf_out, train_out = model(
img, augment=augment) # inference and training outputs
t0 += time_synchronized() - t
# Compute loss
if training: # if model has loss hyperparameters
loss += compute_loss([x.float() for x in train_out], targets,
model)[1][:3] # GIoU, obj, cls
# Run NMS
t = time_synchronized()
output = non_max_suppression(inf_out,
conf_thres=conf_thres,
iou_thres=iou_thres,
merge=merge)
t1 += time_synchronized() - t
# Statistics per image
for si, pred in enumerate(output):
labels = targets[targets[:, 0] == si, 1:]
nl = len(labels)
tcls = labels[:, 0].tolist() if nl else [] # target class
seen += 1
if pred is None:
if nl:
stats.append((torch.zeros(0, niou, dtype=torch.bool),
torch.Tensor(), torch.Tensor(), tcls))
continue
# Append to text file
if save_txt:
gn = torch.tensor(shapes[si][0])[[1, 0, 1, 0
]] # normalization gain whwh
x = pred.clone()
x[:, :4] = scale_coords(img[si].shape[1:], x[:, :4],
shapes[si][0],
shapes[si][1]) # to original
for *xyxy, conf, cls in x:
xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) /
gn).view(-1).tolist() # normalized xywh
with open(str(out / Path(paths[si]).stem) + '.txt',
'a') as f:
f.write(
('%g ' * 5 + '\n') % (cls, *xywh)) # label format
# Clip boxes to image bounds
clip_coords(pred, (height, width))
# Append to pycocotools JSON dictionary
if save_json:
# [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ...
image_id = Path(paths[si]).stem
box = pred[:, :4].clone() # xyxy
scale_coords(img[si].shape[1:], box, shapes[si][0],
shapes[si][1]) # to original shape
box = xyxy2xywh(box) # xywh
box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner
for p, b in zip(pred.tolist(), box.tolist()):
jdict.append({
'image_id':
int(image_id) if image_id.isnumeric() else image_id,
'category_id':
coco91class[int(p[5])],
'bbox': [round(x, 3) for x in b],
'score':
round(p[4], 5)
})
# Assign all predictions as incorrect
correct = torch.zeros(pred.shape[0],
niou,
dtype=torch.bool,
device=device)
if nl:
detected = [] # target indices
tcls_tensor = labels[:, 0]
# target boxes
tbox = xywh2xyxy(labels[:, 1:5]) * whwh
# Per target class
for cls in torch.unique(tcls_tensor):
ti = (cls == tcls_tensor).nonzero(as_tuple=False).view(
-1) # prediction indices
pi = (cls == pred[:, 5]).nonzero(as_tuple=False).view(
-1) # target indices
# Search for detections
if pi.shape[0]:
# Prediction to target ious
ious, i = box_iou(pred[pi, :4], tbox[ti]).max(
1) # best ious, indices
# Append detections
detected_set = set()
for j in (ious > iouv[0]).nonzero(as_tuple=False):
d = ti[i[j]] # detected target
if d.item() not in detected_set:
detected_set.add(d.item())
detected.append(d)
correct[
pi[j]] = ious[j] > iouv # iou_thres is 1xn
if len(
detected
) == nl: # all targets already located in image
break
# Append statistics (correct, conf, pcls, tcls)
stats.append(
(correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls))
# Plot images
if batch_i < 1:
f = Path(save_dir) / ('test_batch%g_gt.jpg' % batch_i) # filename
plot_images(img, targets, paths, str(f), names) # ground truth
f = Path(save_dir) / ('test_batch%g_pred.jpg' % batch_i)
plot_images(img, output_to_target(output, width, height), paths,
str(f), names) # predictions
# Compute statistics
stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy
if len(stats) and stats[0].any():
p, r, ap, f1, ap_class = ap_per_class(*stats)
p, r, ap50, ap = p[:, 0], r[:, 0], ap[:, 0], ap.mean(
1) # [P, R, [email protected], [email protected]:0.95]
mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean()
nt = np.bincount(stats[3].astype(np.int64),
minlength=nc) # number of targets per class
else:
nt = torch.zeros(1)
# Print results
pf = '%20s' + '%12.3g' * 6 # print format
print(pf % ('all', seen, nt.sum(), mp, mr, map50, map))
# Print results per class
if verbose and nc > 1 and len(stats):
for i, c in enumerate(ap_class):
print(pf % (names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i]))
# Print speeds
t = tuple(x / seen * 1E3
for x in (t0, t1, t0 + t1)) + (imgsz, imgsz, batch_size) # tuple
if not training:
print(
'Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g'
% t)
# Save JSON
if save_json and len(jdict):
f = 'detections_val2017_%s_results.json' % \
(weights.split(os.sep)[-1].replace('.pt', '') if isinstance(weights, str) else '') # filename
print('\nCOCO mAP with pycocotools... saving %s...' % f)
with open(f, 'w') as file:
json.dump(jdict, file)
try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
imgIds = [int(Path(x).stem) for x in dataloader.dataset.img_files]
cocoGt = COCO(
glob.glob('../coco/annotations/instances_val*.json')
[0]) # initialize COCO ground truth api
cocoDt = cocoGt.loadRes(f) # initialize COCO pred api
cocoEval = COCOeval(cocoGt, cocoDt, 'bbox')
cocoEval.params.imgIds = imgIds # image IDs to evaluate
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
map, map50 = cocoEval.stats[:
2] # update results ([email protected]:0.95, [email protected])
except Exception as e:
print('ERROR: pycocotools unable to run: %s' % e)
# Return results
model.float() # for training
maps = np.zeros(nc) + map
for i, c in enumerate(ap_class):
maps[c] = ap[i]
return (mp, mr, map50, map,
*(loss.cpu() / len(dataloader)).tolist()), maps, t
def coco_image_segmentation_stats(
seg_mask_output_paths, annotation_paths, seg_mask_image_paths, verbose
):
for (seg_mask_path, annFile, image_path) in zip(
seg_mask_output_paths, annotation_paths, seg_mask_image_paths
):
print("Loading COCO Annotations File: ", annFile)
print("Segmentation Mask Output Folder: ", seg_mask_path)
print("Source Image Folder: ", image_path)
stats_json = os.path.join(seg_mask_path, "image_segmentation_class_stats.json")
print("Image stats will be saved to:", stats_json)
cat_csv = os.path.join(
seg_mask_path, "class_counts_over_sum_category_counts.csv"
)
print("Category weights will be saved to:", cat_csv)
coco = COCO(annFile)
print("Annotation file info:")
coco.info()
print("category ids, not including 0 for background:")
print(coco.getCatIds())
# display COCO categories and supercategories
cats = coco.loadCats(coco.getCatIds())
nms = [cat["name"] for cat in cats]
print("categories: \n\n", " ".join(nms))
nms = set([cat["supercategory"] for cat in cats])
print("supercategories: \n", " ".join(nms))
img_ids = coco.getImgIds()
use_original_dims = True # not target_shape
max_ids = max(ids()) + 1 # add background category
# 0 indicates no category (not even background) for counting bins
max_bin_count = max_ids + 1
bin_count = np.zeros(max_bin_count)
total_pixels = 0
print("Calculating image segmentation stats...")
progbar = Progbar(len(img_ids), verbose=verbose)
i = 0
for idx, img_id in enumerate(img_ids):
img = coco.loadImgs(img_id)[0]
i += 1
progbar.update(i)
ann_ids = coco.getAnnIds(imgIds=img["id"], iscrowd=None)
anns = coco.loadAnns(ann_ids)
target_shape = (img["height"], img["width"], max_ids)
# print('\ntarget_shape:', target_shape)
mask_one_hot = np.zeros(target_shape, dtype=np.uint8)
# Note to only count background pixels once, we define a temporary
# null class of 0, and shift all class category ids up by 1
mask_one_hot[:, :, 0] = 1 # every pixel begins as background
for ann in anns:
mask_partial = coco.annToMask(ann)
above_zero = mask_partial > 0
mask_one_hot[above_zero, ann["category_id"]] = ann["category_id"] + 1
mask_one_hot[above_zero, 0] = 0
# print( mask_one_hot)
# print('initial bin_count shape:', np.shape(bin_count))
# flat_mask_one_hot = mask_one_hot.flatten()
bincount_result = np.bincount(mask_one_hot.flatten())
# print('bincount_result TYPE:', type(bincount_result))
# np.array(np.ndarray.flatten(np.bincount(np.ndarray.
# flatten(np.array(mask_one_hot)).astype(int))).resize(max_bin_count))
# print('bincount_result:', bincount_result)
# print('bincount_result_shape', np.shape(bincount_result))
length = int(np.shape(bincount_result)[0])
zeros_to_add = max_bin_count - length
z = np.zeros(zeros_to_add)
# print('zeros_to_add TYPE:', type(zeros_to_add))
# this is a workaround because for some strange reason the
# output type of bincount couldn't interact with other numpy arrays
bincount_result_long = bincount_result.tolist() + z.tolist()
# bincount_result = bincount_result.resize(max_bin_count)
# print('bincount_result2:', bincount_result_long)
# print('bincount_result2_shape',bincount_result_long)
bin_count = bin_count + np.array(bincount_result_long)
total_pixels += img["height"] * img["width"]
print("Final Tally:")
# shift categories back down by 1
bin_count = bin_count[1:]
category_ids = range(bin_count.size)
sum_category_counts = np.sum(bin_count)
# sum will be =1 as a pixel can be in multiple categories
category_counts_over_sum_category_counts = np.true_divide(
bin_count.astype(np.float64), sum_category_counts
)
np.savetxt(cat_csv, category_counts_over_sum_category_counts)
# sum will be >1 as a pixel can be in multiple categories
category_counts_over_total_pixels = np.true_divide(
bin_count.astype(np.float64), total_pixels
)
# less common categories have more weight, sum = 1
category_counts_p_complement = [
1 - x if x > 0.0 else 0.0 for x in category_counts_over_sum_category_counts
]
# less common categories have more weight, sum > 1
total_pixels_p_complement = [
1 - x if x > 0.0 else 0.0 for x in category_counts_over_total_pixels
]
print(bin_count)
stat_dict = {
"total_pixels": total_pixels,
"category_counts": dict(zip(category_ids, bin_count)),
"sum_category_counts": sum_category_counts,
"category_counts_over_sum_category_counts": dict(
zip(category_ids, category_counts_over_sum_category_counts)
),
"category_counts_over_total_pixels": dict(
zip(category_ids, category_counts_over_total_pixels)
),
"category_counts_p_complement": dict(
zip(category_ids, category_counts_p_complement)
),
"total_pixels_p_complement": dict(
zip(category_ids, total_pixels_p_complement)
),
"ids": ids(),
"categories": categories(),
}
print(stat_dict)
with open(stats_json, "w") as fjson:
json.dump(stat_dict, fjson, ensure_ascii=False)
def coco_json_to_segmentation(
seg_mask_output_paths, annotation_paths, seg_mask_image_paths, verbose
):
for (seg_mask_path, annFile, image_path) in zip(
seg_mask_output_paths, annotation_paths, seg_mask_image_paths
):
print("Loading COCO Annotations File: ", annFile)
print("Segmentation Mask Output Folder: ", seg_mask_path)
print("Source Image Folder: ", image_path)
print(
"\n"
"WARNING: Each pixel can have multiple classes! That means"
"class data overlaps. Also, single objects can be outlined"
"multiple times because they were labeled by different people!"
"In other words, even a single object may be segmented twice."
"This means the .png files are missing entire objects.\n\n"
"Use of categorical one-hot encoded .npy files is recommended,"
"but .npy files also have limitations, because the .npy files"
"only have one label per pixel for each class,"
"and currently take the union of multiple human class labels."
"Improving how your data is handled will improve your results"
"so remember to consider that limitation. There is still"
"an opportunity to improve how this training data is handled &"
"integrated with your training scripts and utilities..."
)
coco = COCO(annFile)
print("Converting Annotations to Segmentation Masks...")
mkdir_p(seg_mask_path)
total_imgs = len(coco.imgToAnns.keys())
progbar = Progbar(total_imgs + len(coco.getImgIds()), verbose=verbose)
# 'annotations' was previously 'instances' in an old version
for img_num in range(total_imgs):
# Both [0]'s are used to extract the element from a list
img = coco.loadImgs(
coco.imgToAnns[coco.imgToAnns.keys()[img_num]][0]["image_id"]
)[0]
h = img["height"]
w = img["width"]
name = img["file_name"]
root_name = name[:-4]
filename = os.path.join(seg_mask_path, root_name + ".png")
file_exists = os.path.exists(filename)
if file_exists:
progbar.update(img_num, [("file_fraction_already_exists", 1)])
continue
else:
progbar.update(img_num, [("file_fraction_already_exists", 0)])
print(filename)
MASK = np.zeros((h, w), dtype=np.uint8)
np.where(MASK > 0)
for ann in coco.imgToAnns[coco.imgToAnns.keys()[img_num]]:
mask = coco.annToMask(ann)
idxs = np.where(mask > 0)
MASK[idxs] = ann["category_id"]
im = Image.fromarray(MASK)
im.save(filename)
print(
"\nConverting Annotations to one hot encoded"
"categorical .npy Segmentation Masks..."
)
img_ids = coco.getImgIds()
use_original_dims = True # not target_shape
for idx, img_id in enumerate(img_ids):
img = coco.loadImgs(img_id)[0]
name = img["file_name"]
root_name = name[:-4]
filename = os.path.join(seg_mask_path, root_name + ".npy")
file_exists = os.path.exists(filename)
if file_exists:
progbar.add(1, [("file_fraction_already_exists", 1)])
continue
else:
progbar.add(1, [("file_fraction_already_exists", 0)])
if use_original_dims:
target_shape = (img["height"], img["width"], max(ids()) + 1)
ann_ids = coco.getAnnIds(imgIds=img["id"], iscrowd=None)
anns = coco.loadAnns(ann_ids)
mask_one_hot = np.zeros(target_shape, dtype=np.uint8)
mask_one_hot[:, :, 0] = 1 # every pixel begins as background
# mask_one_hot = cv2.resize(mask_one_hot,
# target_shape[:2],
# interpolation=cv2.INTER_NEAREST)
for ann in anns:
mask_partial = coco.annToMask(ann)
# mask_partial = cv2.resize(mask_partial,
# (target_shape[1], target_shape[0]),
# interpolation=cv2.INTER_NEAREST)
# # width and height match
# assert mask_one_hot.shape[:2] == mask_partial.shape[:2]
# print('another shape:',
# mask_one_hot[mask_partial > 0].shape)
mask_one_hot[mask_partial > 0, ann["category_id"]] = 1
mask_one_hot[mask_partial > 0, 0] = 0
np.save(filename, mask_one_hot)
def load_data(self):
subject_list = self.get_subject()
sampling_ratio = self.get_subsampling_ratio()
# aggregate annotations from each subject
db = COCO()
for subject in subject_list:
with open(
osp.join(self.annot_path,
'Human36M_subject' + str(subject) + '.json'),
'r') as f:
annot = json.load(f)
if len(db.dataset) == 0:
for k, v in annot.items():
db.dataset[k] = v
else:
for k, v in annot.items():
db.dataset[k] += v
db.createIndex()
if self.data_split == 'test' and not cfg.use_gt_info:
print("Get bounding box and root from " + self.human_bbox_root_dir)
bbox_root_result = {}
with open(self.human_bbox_root_dir) as f:
annot = json.load(f)
for i in range(len(annot)):
bbox_root_result[str(annot[i]['image_id'])] = {
'bbox': np.array(annot[i]['bbox']),
'root': np.array(annot[i]['root_cam'])
}
else:
print("Get bounding box and root from groundtruth")
data = []
for aid in db.anns.keys():
ann = db.anns[aid]
image_id = ann['image_id']
img = db.loadImgs(image_id)[0]
# check subject and frame_idx
subject = img['subject']
frame_idx = img['frame_idx']
if subject not in subject_list:
continue
if frame_idx % sampling_ratio != 0:
continue
img_path = osp.join(self.img_dir, img['file_name'])
img_width, img_height = img['width'], img['height']
cam_param = img['cam_param']
R, t, f, c = np.array(cam_param['R']), np.array(
cam_param['t']), np.array(cam_param['f']), np.array(
cam_param['c'])
# project world coordinate to cam, image coordinate space
joint_cam = np.array(ann['keypoints_cam'])
joint_cam = self.add_thorax(joint_cam)
joint_img = np.zeros((self.joint_num, 3))
joint_img[:, 0], joint_img[:, 1], joint_img[:, 2] = cam2pixel(
joint_cam, f, c)
joint_img[:, 2] = joint_img[:, 2] - joint_cam[self.root_idx, 2]
joint_vis = np.ones((self.joint_num, 1))
if self.data_split == 'test' and not cfg.use_gt_info:
bbox = bbox_root_result[str(
image_id
)]['bbox'] # bbox should be aspect ratio preserved-extended. It is done in RootNet.
root_cam = bbox_root_result[str(image_id)]['root']
else:
bbox = np.array(ann['bbox'])
root_cam = joint_cam[self.root_idx]
# aspect ratio preserving bbox
w = bbox[2]
h = bbox[3]
c_x = bbox[0] + w / 2.
c_y = bbox[1] + h / 2.
aspect_ratio = cfg.input_shape[1] / cfg.input_shape[0]
if w > aspect_ratio * h:
h = w / aspect_ratio
elif w < aspect_ratio * h:
w = h * aspect_ratio
bbox[2] = w * 1.25
bbox[3] = h * 1.25
bbox[0] = c_x - bbox[2] / 2.
bbox[1] = c_y - bbox[3] / 2.
data.append({
'img_path': img_path,
'img_id': image_id,
'bbox': bbox,
'joint_img':
joint_img, # [org_img_x, org_img_y, depth - root_depth]
'joint_cam': joint_cam, # [X, Y, Z] in camera coordinate
'joint_vis': joint_vis,
'root_cam': root_cam, # [X, Y, Z] in camera coordinate
'f': f,
'c': c
})
return data
def load_coco(self,
dataset_dir,
subset,
year=DEFAULT_DATASET_YEAR,
class_ids=None,
class_map=None,
return_coco=False,
auto_download=False):
"""Load a subset of the COCO dataset.
dataset_dir: The root directory of the COCO dataset.
subset: What to load (train, val, minival, valminusminival)
year: What dataset year to load (2014, 2017) as a string, not an integer
class_ids: If provided, only loads images that have the given classes.
class_map: TODO: Not implemented yet. Supports maping classes from
different datasets to the same class ID.
return_coco: If True, returns the COCO object.
auto_download: Automatically download and unzip MS-COCO images and annotations
"""
if auto_download is True:
self.auto_download(dataset_dir, subset, year)
coco = COCO("{}/fashion_train.json".format(dataset_dir))
if subset == "minival" or subset == "valminusminival":
subset = "val"
image_dir = "{}/images/".format(dataset_dir)
# Load all classes or a subset?
if not class_ids:
# All classes
class_ids = sorted(coco.getCatIds())
# All images or a subset?
if class_ids:
image_ids = []
for id in class_ids:
image_ids.extend(list(coco.getImgIds(catIds=[id])))
# Remove duplicates
image_ids = list(set(image_ids))
else:
# All images
image_ids = list(coco.imgs.keys())
# Add classes
for i in class_ids:
self.add_class("coco", i, coco.loadCats(i)[0]["name"])
# Add images
for i in image_ids:
self.add_image("coco",
image_id=i,
path=os.path.join(image_dir,
coco.imgs[i]['file_name']),
width=coco.imgs[i]["width"],
height=coco.imgs[i]["height"],
annotations=coco.loadAnns(
coco.getAnnIds(imgIds=[i],
catIds=class_ids,
iscrowd=None)))
if return_coco:
return coco
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--dir",
"-d",
help="Directory of images to use for comparison",
required=True)
parser.add_argument("--model",
"-m",
help="Path of machine learning model",
required=True)
parser.add_argument("--annotations",
"-a",
help="Path of annotations file",
default="annotations.json")
args = parser.parse_args()
inferencer = TFInferencer(args.model)
images = os.listdir(args.dir)
imgIds = []
for image_name in images:
imgId = int(image_name.split('_')[2].split('.')[0].lstrip('0'))
imgIds.append(imgId)
results = []
for comp_level in COMPRESSION_LEVELS:
for i, image_name in enumerate(images):
img = cv2.imread(os.path.join(args.dir, image_name))
_, img = cv2.imencode("." + ENCODING, img,
[cv2.IMWRITE_JPEG_QUALITY, comp_level])
img = cv2.imdecode(img, 1)
imgId = int(image_name.split('_')[2].split('.')[0].lstrip('0'))
inference = inferencer.inference(img)
for res in inference:
entry = {
"image_id": imgId,
"category_id": int(res[1]),
"bbox": [round(float(x), 2) for x in res[0]],
"score": round(float(res[2]), 3)
}
results.append(entry)
print("Inferenced image", i, "| Compression level", comp_level)
with open("results/results{}.json".format(comp_level), "w") as f:
f.write(json.dumps(results))
inferencer.sess.close()
cocoGT = COCO(args.annotations)
for comp_level in COMPRESSION_LEVELS:
print("COMPRESSION LEVEL", comp_level)
cocoDT = cocoGT.loadRes("results/results{}.json".format(comp_level))
eval = COCOeval(cocoGT, cocoDT, 'bbox')
eval.params.imgIds = imgIds
eval.evaluate()
eval.accumulate()
eval.summarize()
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
from progressbar import progressbar
if __name__ == "__main__":
#coco Ground Truth
cocoGt = COCO("instances_val2017.json")
#Detection results
cocoDt = cocoGt.loadRes("yoloVal.json")
imgIds = sorted(cocoGt.getImgIds())
cocoEval = COCOeval(cocoGt, cocoDt, 'bbox')
cocoEval.params.imgIds = imgIds
cocoEval.evaluate()
cocoEval.accumulate()
print(cocoEval.summarize())
def run_SSDMobileNet_accuracy(engine_file,
batch_size,
num_images,
verbose=False,
output_file="build/out/SSDMobileNet/dump.json"):
logging.info(
"Running SSDMobileNet functionality test for engine [ {:} ] with batch size {:}"
.format(engine_file, batch_size))
runner = EngineRunner(engine_file, verbose=verbose)
input_dtype, input_format = get_input_format(runner.engine)
if input_dtype == trt.DataType.FLOAT:
format_string = "fp32"
elif input_dtype == trt.DataType.INT8:
if input_format == trt.TensorFormat.LINEAR:
format_string = "int8_linear"
elif input_format == trt.TensorFormat.CHW4:
format_string = "int8_chw4"
image_dir = os.path.join(
os.getenv("PREPROCESSED_DATA_DIR", "build/preprocessed_data"),
"coco/val2017/SSDMobileNet", format_string)
annotations_path = os.path.join(
os.getenv("PREPROCESSED_DATA_DIR", "build/preprocessed_data"),
"coco/annotations/instances_val2017.json")
val_map = "data_maps/coco/val_map.txt"
if len(glob(image_dir)) == 0:
logging.warn("Cannot find data directory in ({:})".format(image_dir))
pytest.skip("Cannot find data directory ({:})".format(image_dir))
coco = COCO(annotation_file=annotations_path)
coco_detections = []
image_ids = coco.getImgIds()
num_images = min(num_images, len(image_ids))
logging.info(
"Running validation on {:} images. Please wait...".format(num_images))
batch_idx = 0
for image_idx in range(0, num_images, batch_size):
batch_image_ids = image_ids[image_idx:image_idx + batch_size]
actual_batch_size = len(batch_image_ids)
batch_images = np.ascontiguousarray(
np.stack([
np.load(
os.path.join(image_dir,
coco.imgs[id]["file_name"] + ".npy"))
for id in batch_image_ids
]))
start_time = time.time()
[outputs] = runner([batch_images], actual_batch_size)
if verbose:
logging.info("Batch {:d} >> Inference time: {:f}".format(
batch_idx,
time.time() - start_time))
batch_detections = outputs.reshape(batch_size,
100 * 7 + 1)[:actual_batch_size]
for detections, image_id in zip(batch_detections, batch_image_ids):
keep_count = detections[100 * 7].view('int32')
image_width = coco.imgs[image_id]["width"]
image_height = coco.imgs[image_id]["height"]
for detection in detections[:keep_count * 7].reshape(
keep_count, 7):
score = float(detection[PredictionLayout.CONFIDENCE])
bbox_coco_fmt = [
detection[PredictionLayout.XMIN] * image_width,
detection[PredictionLayout.YMIN] * image_height,
(detection[PredictionLayout.XMAX] -
detection[PredictionLayout.XMIN]) * image_width,
(detection[PredictionLayout.YMAX] -
detection[PredictionLayout.YMIN]) * image_height,
]
coco_detection = {
"image_id": image_id,
"category_id": int(detection[PredictionLayout.LABEL]),
"bbox": bbox_coco_fmt,
"score": score,
}
coco_detections.append(coco_detection)
batch_idx += 1
output_dir = os.path.dirname(output_file)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
with open(output_file, "w") as f:
json.dump(coco_detections, f)
cocoDt = coco.loadRes(output_file)
eval = COCOeval(coco, cocoDt, 'bbox')
eval.params.imgIds = image_ids[:num_images]
eval.evaluate()
eval.accumulate()
eval.summarize()
map_score = eval.stats[0]
logging.info("Get mAP score = {:f} Target = {:f}".format(
map_score, 0.22386))
return map_score
from pycocotools.coco import COCO
import numpy as np
import skimage.io as io
import matplotlib.pyplot as plt
import pylab
import json
pylab.rcParams['figure.figsize'] = (8.0, 10.0)
dataDir = '../cocoData'
dataType = 'val2017'
annFile = '{}/annotations/instances_{}.json'.format(dataDir, dataType)
coco = COCO(annFile)
# display COCO categories and supercategories
cats = coco.loadCats(coco.getCatIds())
nms = [cat['name'] for cat in cats]
# print('COCO categories: \n{}\n'.format(' '.join(nms)))
nms = set([cat['supercategory'] for cat in cats])
# print('COCO supercategories: \n{}'.format(' '.join(nms)))
catIds = coco.getCatIds(catNms=['person', 'car', 'bus'])
imgIds = coco.getImgIds(catIds=catIds)
annIds = coco.getAnnIds(imgIds, catIds)
loadAnns = coco.loadAnns(annIds)
loadCats = coco.loadCats(catIds)
loadImgs = coco.loadImgs(imgIds)
# print(newImages)
newData = {'images': loadImgs, 'annotations': loadAnns, 'categories': loadCats}
return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))
def getexample(im, bboxes, Counts, Id):
return tf.train.Example(features=tf.train.Features(feature={
'im': _bytes([im]),
'bboxes': _bytes([bboxes]),
'Counts': _bytes(Counts),
'Id': _bytes([Id]),
}))
train_file = '/home/zhai/PycharmProjects/Demo35/dataset/coco/instances_valminusminival2014.json'
train_file = '/home/zhai/PycharmProjects/Demo35/dataset/coco/annotations/instances_train2017.json'
cocoGt = COCO(train_file)
imgIds = cocoGt.getImgIds()
imgIds = sorted(imgIds)
catIds = cocoGt.getCatIds()
print(type(catIds))
catId2cls = {}
cls2catId = {}
catId2name = {}
name2cls = {}
name2catId = {}
cls2name = {}
for i in range(len(catIds)):
catId2cls[catIds[i]] = i
cls2catId[i] = catIds[i]
with open(os.path.join(main_path, 'rval_train.json'), 'w') as outfile:
json.dump(trainrestval, outfile)
with open(os.path.join(main_path, 'rval_val.json'), 'w') as outfile:
json.dump(valid, outfile)
with open(os.path.join(main_path, 'rval_test.json'), 'w') as outfile:
json.dump(test, outfile)
with open(os.path.join(main_path, 'rval_train_only.json'), 'w') as outfile:
json.dump(test, outfile)
print('1,2 are finished')
# 3. make 2017 data list
train = []
valid = []
jf = json.load(open(os.path.join(main_path, 'captions_train2017.json')))['images']
caps = COCO(os.path.join(main_path, 'captions_train2017.json'))
for i in range(len(jf)):
ann_ids = caps.getAnnIds(imgIds=jf[i]['id'])
data = {'sentid':ann_ids,
'imgid':jf[i]['id'],
'filepath':'train2017',
'filename':jf[i]['file_name'],}
train.append(data)
if i % 3000 == 0:
print('train2017-{}th data are processed'.format(i))
jf = json.load(open(os.path.join(main_path, 'captions_val2017.json')))['images']
caps = COCO(os.path.join(main_path, 'captions_val2017.json'))
for i in range(len(jf)):
val_loader = torch.utils.data.DataLoader(
validation_dataset,
batch_size=BATCH_SIZE,
num_workers=6,
shuffle=False,
sampler=SequentialSampler(validation_dataset),
pin_memory=False,
collate_fn=collate_fn,
)
device = torch.device('cuda') if torch.cuda.is_available() else torch.device(
'cpu')
print('Cuda is available: {}'.format(torch.cuda.is_available()))
cpu_device = torch.device("cpu")
annType = 'bbox'
cocoGt = COCO(os.path.join(DATASET_DIR, 'ground-truth-test.json'))
num_classes = 2
EPOCHS = 300
CHECKPOINT_FREQ = 1
model = torchvision.models.detection.fasterrcnn_resnet50_fpn(
pretrained=True, pretrained_backbone=True)
in_features = model.roi_heads.box_predictor.cls_score.in_features
model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)
# if torch.cuda.device_count() > 1:
# print("Let's use", torch.cuda.device_count(), "GPUs!")
# # dim = 0 [30, xxx] -> [10, ...], [10, ...], [10, ...] on 3 GPUs
# model = torch.nn.DataParallel(model)
model.to(device)
def __init__(self, opt, split='train', seq_per_img=5):
self.opt = opt
self.batch_size = self.opt.batch_size
self.seq_per_img = opt.seq_per_img
self.seq_length = opt.seq_length
self.split = split
self.seq_per_img = seq_per_img
# image processing function.
if split == 'train':
self.Resize = transforms.Resize(
(self.opt.image_size, self.opt.image_size))
else:
self.Resize = transforms.Resize(
(self.opt.image_crop_size, self.opt.image_crop_size))
self.RandomCropWithBbox = utils.RandomCropWithBbox(opt.image_crop_size)
self.ToTensor = transforms.ToTensor()
self.res_Normalize = transforms.Normalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])
self.vgg_pixel_mean = np.array([[[103.939, 116.779, 123.68]]])
self.max_gt_box = 100
self.max_proposal = 200
self.glove = vocab.GloVe(name='6B', dim=300)
if opt.det_oracle == True:
print('Training and Inference under oracle Mode...')
# load the json file which contains additional information about the dataset
print('DataLoader loading json file: ', opt.input_dic)
self.info = json.load(open(self.opt.input_dic))
self.itow = self.info['ix_to_word']
self.wtoi = {w: i for i, w in self.itow.items()}
self.wtod = {w: i + 1
for w, i in self.info['wtod'].items()
} # word to detection
self.dtoi = {w: i + 1
for i, w in enumerate(self.wtod.keys())
} # detection to index
self.itod = {i + 1: w for i, w in enumerate(self.wtod.keys())}
self.wtol = self.info['wtol']
self.ltow = {l: w for w, l in self.wtol.items()}
self.vocab_size = len(self.itow) + 1 # since it start from 1
print('vocab size is ', self.vocab_size)
# initilize the fg+s/p map back to word idx.
self.st2towidx = np.zeros(len(self.dtoi) * 2 +
1) # statge 2 to word idex
for w, i in self.dtoi.items():
s2_idx = i * 2 - 1
if w not in self.wtoi:
w = 'UNK'
w_idx = self.wtoi[w]
self.st2towidx[s2_idx] = w_idx
# get the plural idx.
if w in self.ltow:
pw = self.ltow[w]
w_idx = self.wtoi[pw]
self.st2towidx[s2_idx + 1] = w_idx
# get the glove vector for the fg detections.
self.glove_fg = np.zeros((len(self.dtoi) + 1, 300))
for i, word in enumerate(self.dtoi.keys()):
vector = np.zeros((300))
count = 0
for w in word.split(' '):
count += 1
if w in self.glove.stoi:
glove_vector = self.glove.vectors[self.glove.stoi[w]]
vector += glove_vector.numpy()
else: # use a random vector instead
random_vector = 2 * np.random.rand(300) - 1
vector += random_vector
self.glove_fg[i + 1] = vector / count
self.glove_w = np.zeros((len(self.wtoi) + 1, 300))
for i, word in enumerate(self.wtoi.keys()):
vector = np.zeros((300))
count = 0
for w in word.split(' '):
count += 1
if w in self.glove.stoi:
glove_vector = self.glove.vectors[self.glove.stoi[w]]
vector += glove_vector.numpy()
else: # use a random vector instead
random_vector = 2 * np.random.rand(300) - 1
vector += random_vector
self.glove_w[i + 1] = vector / count
# open the caption json file
print('DataLoader loading json file: ', opt.input_json)
self.caption_file = json.load(open(self.opt.input_json))
# open the detection json file.
self.dataloader_hdf = HDFSingleDataset(self.opt.proposal_h5)
# load the coco grounding truth bounding box.
det_train_path = '%s/coco/annotations/instances_train2014.json' % (
opt.data_path)
det_val_path = '%s/coco/annotations/instances_val2014.json' % (
opt.data_path)
self.coco_train = COCO(det_train_path)
self.coco_val = COCO(det_val_path)
# category id to labels. +1 becuase 0 is the background label.
self.ctol = {c: i + 1 for i, c in enumerate(self.coco_val.cats.keys())}
self.itoc = {
i + 1: c['name']
for i, c in enumerate(self.coco_val.cats.values())
}
self.ctoi = {c: i for i, c in self.itoc.items()}
self.glove_clss = np.zeros((len(self.itoc) + 1, 300))
for i, word in enumerate(self.itoc.values()):
vector = np.zeros((300))
count = 0
# if we decode novel word, replace the word representation based on the dictionary.
if opt.decode_noc and word in utils.noc_word_map:
word = utils.noc_word_map[word]
for w in word.split(' '):
count += 1
if w in self.glove.stoi:
glove_vector = self.glove.vectors[self.glove.stoi[w]]
vector += glove_vector.numpy()
else: # use a random vector instead
random_vector = 2 * np.random.rand(300) - 1
vector += random_vector
self.glove_clss[i + 1] = vector / count
self.detect_size = len(self.ctol)
self.fg_size = len(self.dtoi)
# get the fine-grained mask.
self.fg_mask = np.ones((self.detect_size + 1, self.fg_size + 1))
for w, det in self.wtod.items():
self.fg_mask[det, self.dtoi[w]] = 0
# separate out indexes for each of the provided splits
self.split_ix = []
for ix in range(len(self.info['images'])):
img = self.info['images'][ix]
if img['split'] == split:
self.split_ix.append(ix)
print('assigned %d images to split %s' % (len(self.split_ix), split))
def load_coco_json(json_file, image_root, dataset_name=None, extra_annotation_keys=None):
"""
Load a json file with COCO's instances annotation format.
Currently supports instance detection, instance segmentation,
and person keypoints annotations.
Args:
json_file (str): full path to the json file in COCO instances annotation format.
image_root (str or path-like): the directory where the images in this json file exists.
dataset_name (str): the name of the dataset (e.g., coco_2017_train).
If provided, this function will also put "thing_classes" into
the metadata associated with this dataset.
extra_annotation_keys (list[str]): list of per-annotation keys that should also be
loaded into the dataset dict (besides "iscrowd", "bbox", "keypoints",
"category_id", "segmentation"). The values for these keys will be returned as-is.
For example, the densepose annotations are loaded in this way.
Returns:
list[dict]: a list of dicts in Detectron2 standard dataset dicts format. (See
`Using Custom Datasets </tutorials/datasets.html>`_ )
Notes:
1. This function does not read the image files.
The results do not have the "image" field.
"""
from pycocotools.coco import COCO
timer = Timer()
json_file = PathManager.get_local_path(json_file)
with contextlib.redirect_stdout(io.StringIO()):
coco_api = COCO(json_file)
if timer.seconds() > 1:
logger.info("Loading {} takes {:.2f} seconds.".format(json_file, timer.seconds()))
id_map = None
if dataset_name is not None:
meta = MetadataCatalog.get(dataset_name)
cat_ids = sorted(coco_api.getCatIds())
cats = coco_api.loadCats(cat_ids)
# The categories in a custom json file may not be sorted.
thing_classes = [c["name"] for c in sorted(cats, key=lambda x: x["id"])]
meta.thing_classes = thing_classes
# In COCO, certain category ids are artificially removed,
# and by convention they are always ignored.
# We deal with COCO's id issue and translate
# the category ids to contiguous ids in [0, 80).
# It works by looking at the "categories" field in the json, therefore
# if users' own json also have incontiguous ids, we'll
# apply this mapping as well but print a warning.
if not (min(cat_ids) == 1 and max(cat_ids) == len(cat_ids)):
if "coco" not in dataset_name:
logger.warning(
"""
Category ids in annotations are not in [1, #categories]! We'll apply a mapping for you.
"""
)
id_map = {v: i for i, v in enumerate(cat_ids)}
meta.thing_dataset_id_to_contiguous_id = id_map
# sort indices for reproducible results
img_ids = sorted(coco_api.imgs.keys())
# imgs is a list of dicts, each looks something like:
# {'license': 4,
# 'url': 'http://farm6.staticflickr.com/5454/9413846304_881d5e5c3b_z.jpg',
# 'file_name': 'COCO_val2014_000000001268.jpg',
# 'height': 427,
# 'width': 640,
# 'date_captured': '2013-11-17 05:57:24',
# 'id': 1268}
imgs = coco_api.loadImgs(img_ids)
# anns is a list[list[dict]], where each dict is an annotation
# record for an object. The inner list enumerates the objects in an image
# and the outer list enumerates over images. Example of anns[0]:
# [{'segmentation': [[192.81,
# 247.09,
# ...
# 219.03,
# 249.06]],
# 'area': 1035.749,
# 'iscrowd': 0,
# 'image_id': 1268,
# 'bbox': [192.81, 224.8, 74.73, 33.43],
# 'category_id': 16,
# 'id': 42986},
# ...]
anns = [coco_api.imgToAnns[img_id] for img_id in img_ids]
total_num_valid_anns = sum([len(x) for x in anns])
total_num_anns = len(coco_api.anns)
if total_num_valid_anns < total_num_anns:
logger.warning(
f"{json_file} contains {total_num_anns} annotations, but only "
f"{total_num_valid_anns} of them match to images in the file."
)
if "minival" not in json_file:
# The popular valminusminival & minival annotations for COCO2014 contain this bug.
# However the ratio of buggy annotations there is tiny and does not affect accuracy.
# Therefore we explicitly white-list them.
ann_ids = [ann["id"] for anns_per_image in anns for ann in anns_per_image]
assert len(set(ann_ids)) == len(ann_ids), "Annotation ids in '{}' are not unique!".format(
json_file
)
imgs_anns = list(zip(imgs, anns))
logger.info("Loaded {} images in COCO format from {}".format(len(imgs_anns), json_file))
dataset_dicts = []
ann_keys = ["iscrowd", "bbox", "keypoints", "category_id"] + (extra_annotation_keys or [])
num_instances_without_valid_segmentation = 0
for (img_dict, anno_dict_list) in imgs_anns:
record = {}
record["file_name"] = os.path.join(image_root, img_dict["file_name"])
record["height"] = img_dict["height"]
record["width"] = img_dict["width"]
image_id = record["image_id"] = img_dict["id"]
objs = []
for anno in anno_dict_list:
# Check that the image_id in this annotation is the same as
# the image_id we're looking at.
# This fails only when the data parsing logic or the annotation file is buggy.
# The original COCO valminusminival2014 & minival2014 annotation files
# actually contains bugs that, together with certain ways of using COCO API,
# can trigger this assertion.
assert anno["image_id"] == image_id
assert anno.get("ignore", 0) == 0, '"ignore" in COCO json file is not supported.'
obj = {key: anno[key] for key in ann_keys if key in anno}
segm = anno.get("segmentation", None)
if segm: # either list[list[float]] or dict(RLE)
if isinstance(segm, dict):
if isinstance(segm["counts"], list):
# convert to compressed RLE
segm = mask_util.frPyObjects(segm, *segm["size"])
else:
# filter out invalid polygons (< 3 points)
segm = [poly for poly in segm if len(poly) % 2 == 0 and len(poly) >= 6]
if len(segm) == 0:
num_instances_without_valid_segmentation += 1
continue # ignore this instance
obj["segmentation"] = segm
keypts = anno.get("keypoints", None)
if keypts: # list[int]
for idx, v in enumerate(keypts):
if idx % 3 != 2:
# COCO's segmentation coordinates are floating points in [0, H or W],
# but keypoint coordinates are integers in [0, H-1 or W-1]
# Therefore we assume the coordinates are "pixel indices" and
# add 0.5 to convert to floating point coordinates.
keypts[idx] = v + 0.5
obj["keypoints"] = keypts
obj["bbox_mode"] = BoxMode.XYWH_ABS
if id_map:
obj["category_id"] = id_map[obj["category_id"]]
objs.append(obj)
record["annotations"] = objs
dataset_dicts.append(record)
if num_instances_without_valid_segmentation > 0:
logger.warning(
"Filtered out {} instances without valid segmentation. ".format(
num_instances_without_valid_segmentation
)
+ "There might be issues in your dataset generation process. "
"A valid polygon should be a list[float] with even length >= 6."
)
return dataset_dicts
def load_data(self):
if self.data_split == 'train':
db = COCO(self.annot_path)
with open(self.smpl_param_path) as f:
smpl_params = json.load(f)
else:
print('Unknown data subset')
assert 0
datalist = []
for iid in db.imgs.keys():
img = db.imgs[iid]
img_id = img["id"]
img_width, img_height = img['width'], img['height']
imgname = img['file_name']
img_path = osp.join(self.img_dir, imgname)
focal = img["f"]
princpt = img["c"]
cam_param = {'focal': focal, 'princpt': princpt}
# crop the closest person to the camera
ann_ids = db.getAnnIds(img_id)
anns = db.loadAnns(ann_ids)
# sample closest subject
root_depths = [
ann['keypoints_cam'][self.muco_root_joint_idx][2]
for ann in anns
]
closest_pid = root_depths.index(min(root_depths))
pid_list = [closest_pid]
# sample close subjects
# for i in range(len(anns)):
# if i == closest_pid:
# continue
# picked = True
# for j in range(len(anns)):
# if i == j:
# continue
# dist = (np.array(anns[i]['keypoints_cam'][self.muco_root_joint_idx]) - np.array(
# anns[j]['keypoints_cam'][self.muco_root_joint_idx])) ** 2
# dist_2d = math.sqrt(np.sum(dist[:2]))
# dist_3d = math.sqrt(np.sum(dist))
# if dist_2d < 500 or dist_3d < 500:
# picked = False
# if picked:
# pid_list.append(i)
for pid in pid_list:
joint_cam = np.array(anns[pid]['keypoints_cam'])
joint_img = np.array(anns[pid]['keypoints_img'])
joint_img = np.concatenate([joint_img, joint_cam[:, 2:]], 1)
joint_valid = np.ones((self.muco_joint_num, 3))
bbox = process_bbox(anns[pid]['bbox'])
if bbox is None: continue
# check smpl parameter exist
try:
smpl_param = smpl_params[str(ann_ids[pid])]
pose, shape, trans = np.array(
smpl_param['pose']), np.array(
smpl_param['shape']), np.array(smpl_param['trans'])
sum = pose.sum() + shape.sum() + trans.sum()
if np.isnan(sum):
continue
except KeyError:
continue
datalist.append({
'annot_id': ann_ids[pid],
'img_path': img_path,
'img_shape': (img_height, img_width),
'bbox': bbox,
'joint_img': joint_img,
'joint_cam': joint_cam,
'joint_valid': joint_valid,
'cam_param': cam_param,
'smpl_param': smpl_param
})
if self.debug and len(datalist) > 10000:
break
return datalist
def test(data,
weights=None,
batch_size=32,
imgsz=640,
conf_thres=0.1,
iou_thres=0.5, # for NMS
save_json=False,
single_cls=False,
augment=False,
verbose=False,
model=None,
dataloader=None,
save_dir=Path(''), # for saving images
save_txt=False, # for auto-labelling
save_conf=False,
plots=True,
log_imgs=0): # number of logged images
# Initialize/load model and set device
training = model is not None
if training: # called by train.py
device = next(model.parameters()).device # get model device
else: # called directly
set_logging()
device = select_device(opt.device, batch_size=batch_size)
save_txt = opt.save_txt # save *.txt labels
# Directories
save_dir = Path(increment_path(Path(opt.project) / opt.name, exist_ok=opt.exist_ok)) # increment run
(save_dir / 'labels' if save_txt else save_dir).mkdir(parents=True, exist_ok=True) # make dir
# Load model
model = attempt_load(weights, map_location=device) # load FP32 model
imgsz = check_img_size(imgsz, s=model.stride.max()) # check img_size
# Multi-GPU disabled, incompatible with .half() https://github.com/ultralytics/yolov5/issues/99
# if device.type != 'cpu' and torch.cuda.device_count() > 1:
# model = nn.DataParallel(model)
# Half
half = device.type != 'cpu' # half precision only supported on CUDA
if half:
model.half()
# Configure
model.eval()
is_coco = data.endswith('coco.yaml') # is COCO dataset
with open(data) as f:
data = yaml.load(f, Loader=yaml.FullLoader) # model dict
check_dataset(data) # check
nc = 1 if single_cls else int(data['nc']) # number of classes
iouv = torch.linspace(0.5, 0.95, 10).to(device) # iou vector for [email protected]:0.95
niou = iouv.numel()
# Logging
log_imgs, wandb = min(log_imgs, 100), None # ceil
try:
import wandb # Weights & Biases
except ImportError:
log_imgs = 0
# Dataloader
if not training:
img = torch.zeros((1, 3, imgsz, imgsz), device=device) # init img
_ = model(img.half() if half else img) if device.type != 'cpu' else None # run once
path = data['test'] if opt.task == 'test' else data['val'] # path to val/test images
dataloader = create_dataloader(path, imgsz, batch_size, model.stride.max(), opt, pad=0.5, rect=True)[0]
seen = 0
names = {k: v for k, v in enumerate(model.names if hasattr(model, 'names') else model.module.names)}
coco91class = coco80_to_coco91_class()
s = ('%20s' + '%12s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R', '[email protected]', '[email protected]:.95')
p, r, f1, mp, mr, map50, map, t0, t1 = 0., 0., 0., 0., 0., 0., 0., 0., 0.
loss = torch.zeros(3, device=device)
jdict, stats, ap, ap_class, wandb_images = [], [], [], [], []
for batch_i, (img, targets, paths, shapes) in enumerate(tqdm(dataloader, desc=s)):
img = img.to(device, non_blocking=True)
img = img.half() if half else img.float() # uint8 to fp16/32
img /= 255.0 # 0 - 255 to 0.0 - 1.0
targets = targets.to(device)
nb, _, height, width = img.shape # batch size, channels, height, width
whwh = torch.Tensor([width, height, width, height]).to(device)
# Disable gradients
with torch.no_grad():
# Run model
t = time_synchronized()
inf_out, train_out = model(img, augment=augment) # inference and training outputs
t0 += time_synchronized() - t
# Compute loss
if training: # if model has loss hyperparameters
loss += compute_loss([x.float() for x in train_out], targets, model)[1][:3] # box, obj, cls
# Run NMS
t = time_synchronized()
output = non_max_suppression(inf_out, conf_thres=conf_thres, iou_thres=iou_thres)
t1 += time_synchronized() - t
# Statistics per image
for si, pred in enumerate(output):
labels = targets[targets[:, 0] == si, 1:]
nl = len(labels)
tcls = labels[:, 0].tolist() if nl else [] # target class
path = Path(paths[si])
seen += 1
if len(pred) == 0:
if nl:
stats.append((torch.zeros(0, niou, dtype=torch.bool), torch.Tensor(), torch.Tensor(), tcls))
continue
# Predictions
predn = pred.clone()
scale_coords(img[si].shape[1:], predn[:, :4], shapes[si][0], shapes[si][1]) # native-space pred
# Append to text file
if save_txt:
gn = torch.tensor(shapes[si][0])[[1, 0, 1, 0]] # normalization gain whwh
for *xyxy, conf, cls in predn.tolist():
xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh
line = (cls, *xywh, conf) if save_conf else (cls, *xywh) # label format
with open(save_dir / 'labels' / (path.stem + '.txt'), 'a') as f:
f.write(('%g ' * len(line)).rstrip() % line + '\n')
# W&B logging
if plots and len(wandb_images) < log_imgs:
box_data = [{"position": {"minX": xyxy[0], "minY": xyxy[1], "maxX": xyxy[2], "maxY": xyxy[3]},
"class_id": int(cls),
"box_caption": "%s %.3f" % (names[cls], conf),
"scores": {"class_score": conf},
"domain": "pixel"} for *xyxy, conf, cls in pred.tolist()]
boxes = {"predictions": {"box_data": box_data, "class_labels": names}} # inference-space
wandb_images.append(wandb.Image(img[si], boxes=boxes, caption=path.name))
# Append to pycocotools JSON dictionary
if save_json:
# [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ...
image_id = int(path.stem) if path.stem.isnumeric() else path.stem
box = xyxy2xywh(predn[:, :4]) # xywh
box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner
for p, b in zip(pred.tolist(), box.tolist()):
jdict.append({'image_id': image_id,
'category_id': coco91class[int(p[5])] if is_coco else int(p[5]),
'bbox': [round(x, 3) for x in b],
'score': round(p[4], 5)})
# Assign all predictions as incorrect
correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool, device=device)
if nl:
detected = [] # target indices
tcls_tensor = labels[:, 0]
# target boxes
tbox = xywh2xyxy(labels[:, 1:5]) * whwh
scale_coords(img[si].shape[1:], tbox, shapes[si][0], shapes[si][1]) # native-space labels
# Per target class
for cls in torch.unique(tcls_tensor):
ti = (cls == tcls_tensor).nonzero(as_tuple=False).view(-1) # prediction indices
pi = (cls == pred[:, 5]).nonzero(as_tuple=False).view(-1) # target indices
# Search for detections
if pi.shape[0]:
# Prediction to target ious
ious, i = box_iou(predn[pi, :4], tbox[ti]).max(1) # best ious, indices
# Append detections
detected_set = set()
for j in (ious > iouv[0]).nonzero(as_tuple=False):
d = ti[i[j]] # detected target
if d.item() not in detected_set:
detected_set.add(d.item())
detected.append(d)
correct[pi[j]] = ious[j] > iouv # iou_thres is 1xn
if len(detected) == nl: # all targets already located in image
break
# Append statistics (correct, conf, pcls, tcls)
stats.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls))
# Plot images
if plots and batch_i < 3:
f = save_dir / f'test_batch{batch_i}_labels.jpg' # filename
plot_images(img, targets, paths, f, names) # labels
f = save_dir / f'test_batch{batch_i}_pred.jpg'
plot_images(img, output_to_target(output, width, height), paths, f, names) # predictions
# Compute statistics
stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy
if len(stats) and stats[0].any():
p, r, ap, f1, ap_class = ap_per_class(*stats, plot=plots, save_dir=save_dir, names=names)
p, r, ap50, ap = p[:, 0], r[:, 0], ap[:, 0], ap.mean(1) # [P, R, [email protected], [email protected]:0.95]
mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean()
nt = np.bincount(stats[3].astype(np.int64), minlength=nc) # number of targets per class
else:
nt = torch.zeros(1)
# W&B logging
if plots and wandb and wandb.run:
wandb.log({"Images": wandb_images})
wandb.log({"Validation": [wandb.Image(str(x), caption=x.name) for x in sorted(save_dir.glob('test*.jpg'))]})
# Print results
pf = '%20s' + '%12.3g' * 6 # print format
print(pf % ('all', seen, nt.sum(), mp, mr, map50, map))
# Print results per class
if verbose and nc > 1 and len(stats):
for i, c in enumerate(ap_class):
print(pf % (names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i]))
# Print speeds
t = tuple(x / seen * 1E3 for x in (t0, t1, t0 + t1)) + (imgsz, imgsz, batch_size) # tuple
if not training:
print('Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g' % t)
# Save JSON
if save_json and len(jdict):
w = Path(weights[0] if isinstance(weights, list) else weights).stem if weights is not None else '' # weights
anno_json = glob.glob('../coco/annotations/instances_val*.json')[0] # annotations json
pred_json = str(save_dir / f"{w}_predictions.json") # predictions json
print('\nEvaluating pycocotools mAP... saving %s...' % pred_json)
with open(pred_json, 'w') as f:
json.dump(jdict, f)
try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
anno = COCO(anno_json) # init annotations api
pred = anno.loadRes(pred_json) # init predictions api
eval = COCOeval(anno, pred, 'bbox')
if is_coco:
eval.params.imgIds = [int(Path(x).stem) for x in dataloader.dataset.img_files] # image IDs to evaluate
eval.evaluate()
eval.accumulate()
eval.summarize()
map, map50 = eval.stats[:2] # update results ([email protected]:0.95, [email protected])
except Exception as e:
print('ERROR: pycocotools unable to run: %s' % e)
# Return results
if not training:
print('Results saved to %s' % save_dir)
model.float() # for training
maps = np.zeros(nc) + map
for i, c in enumerate(ap_class):
maps[c] = ap[i]
return (mp, mr, map50, map, *(loss.cpu() / len(dataloader)).tolist()), maps, t
np.savez_compressed('train_features',
img_features=img_features,
tag_features=tag_features)
logging.info('Training: number of possible tags = %d', len(possible_tags))
pickle.dump(possible_tags, open('possible_tags.pkl', 'wb'))
if __name__ == "__main__":
logging.basicConfig(filename='cca.log',
format='%(asctime)s %(message)s',
level=logging.INFO)
parser = argparse.ArgumentParser()
parser.add_argument('--tagsPerImage',
default=2,
type=int,
help='amount of tags per image')
args = parser.parse_args()
annFile = 'annotations/captions_train2014.json'
coco_train = COCO(annFile)
ids = coco_train.getAnnIds()
annotations = coco_train.loadAnns(ids)
img_count = {}
img_captions = {}
count_words()
calc_features()
Download COCO 2014 train/val annotations: http://cocodataset.org/#download
Make sure to change the data directory below to wherever you store the annotation data.
Follow these steps to install pycocotools.coco: https://github.com/cocodataset/cocoapi
You can simply cd into PythonAPI directory and run "make"
Probably don't need to look at this unless you want to alter the data accessed.
COCO api: https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoDemo.ipynb
"""
dataDir = '../../..'
dataType = 'train2014'
annotations = '{}/annotations/instances_{}.json'.format(dataDir, dataType)
captions = '{}/annotations/captions_{}.json'.format(dataDir, dataType)
coco_anns = COCO(annotations)
coco_caps = COCO(captions)
male_biased = ['truck', 'motorcycle', 'tie', 'backpack', 'sports ball']
# some of these are actually male biased due to lack of female biased terms
female_biased = [
'handbag', 'fork', 'knife', 'spoon', 'cell phone', 'teddy bear'
]
bias_terms = female_biased + male_biased
training_set = {} # dict containing imgIds: labels
male_count = 0
female_count = 0
for term in bias_terms:
catIds = coco_anns.getCatIds(catNms=['person', term])
def loadRes(self, resFile):
"""
Load result file and return a result api object.
:param resFile (str) : file name of result file
:return: res (obj) : result api object
"""
res = COCO()
res.dataset['images'] = [img for img in self.dataset['images']]
# print('Loading and preparing results...')
# tic = time.time()
if isinstance(resFile, torch._six.string_classes):
anns = json.load(open(resFile))
elif type(resFile) == np.ndarray:
anns = self.loadNumpyAnnotations(resFile)
else:
anns = resFile
assert type(anns) == list, 'results in not an array of objects'
annsImgIds = [ann['image_id'] for ann in anns]
assert set(annsImgIds) == (set(annsImgIds) & set(self.getImgIds())), \
'Results do not correspond to current coco set'
if 'caption' in anns[0]:
imgIds = set([img['id'] for img in res.dataset['images']]) & set(
[ann['image_id'] for ann in anns])
res.dataset['images'] = [
img for img in res.dataset['images'] if img['id'] in imgIds
]
for id, ann in enumerate(anns):
ann['id'] = id + 1
elif 'bbox' in anns[0] and not anns[0]['bbox'] == []:
res.dataset['categories'] = copy.deepcopy(self.dataset['categories'])
for id, ann in enumerate(anns):
bb = ann['bbox']
x1, x2, y1, y2 = [bb[0], bb[0] + bb[2], bb[1], bb[1] + bb[3]]
if 'segmentation' not in ann:
ann['segmentation'] = [[x1, y1, x1, y2, x2, y2, x2, y1]]
ann['area'] = bb[2] * bb[3]
ann['id'] = id + 1
ann['iscrowd'] = 0
elif 'segmentation' in anns[0]:
res.dataset['categories'] = copy.deepcopy(self.dataset['categories'])
for id, ann in enumerate(anns):
# now only support compressed RLE format as segmentation results
ann['area'] = maskUtils.area(ann['segmentation'])
if 'bbox' not in ann:
ann['bbox'] = maskUtils.toBbox(ann['segmentation'])
ann['id'] = id + 1
ann['iscrowd'] = 0
elif 'keypoints' in anns[0]:
res.dataset['categories'] = copy.deepcopy(self.dataset['categories'])
for id, ann in enumerate(anns):
s = ann['keypoints']
x = s[0::3]
y = s[1::3]
x1, x2, y1, y2 = np.min(x), np.max(x), np.min(y), np.max(y)
ann['area'] = (x2 - x1) * (y2 - y1)
ann['id'] = id + 1
ann['bbox'] = [x1, y1, x2 - x1, y2 - y1]
# print('DONE (t={:0.2f}s)'.format(time.time()- tic))
res.dataset['annotations'] = anns
createIndex(res)
return res
from pycocotools.coco import COCO
coco = COCO("./all.json")
print(coco.getCatIds())
# print()
