def task3():
algorithms = ['KNN', 'MOG2', 'CNT', 'GMG', 'GSOC', 'MOG'] #'LSBP'
for i in range(len(algorithms)):
# Read gt file
path = "ai_challenge_s03_c010-full_annotation.xml"
reader = ReadData(path)
model = BgsModel(
path='/home/mar/Desktop/M6/Lab1/AICity_data/train/S03/c010/vdo.avi',
color_space='gray',
alg=algorithms[i])
vidLen = model.retVidLen()
gt, num_iter = reader.getGTfromXML()
gt = reader.preprocessGT(gt)
sortedFrames, sortedBBOX, numBBOX = reader.bboxInFrame(
gt, int(vidLen * 0.25))
gtInfo = reader.joinBBOXfromFrame(sortedFrames, sortedBBOX, isGT=True)
predictionsInfo, num_bboxes = model.foreground_extraction(
showVid=True, gt=gtInfo, use_postprocessing=True)
# Uncomment if original video with bboxes is needed:
# video_with_bbox("D:\MCV\M6\AICity_data\train\S03\c010\vdo.avi", algorithms[i], gtInfo, predictionsInfo, int(vidLen*0.25), int(vidLen))
rec, prec, ap, meanIoU = ap_score(gtInfo,
predictionsInfo,
num_bboxes=num_bboxes,
ovthresh=0.5)
print('Method: ', algorithms[i])
print('mAP:', ap)
print('Mean IoU:', meanIoU)
def task11():
#Read gt file
path = 'ai_challenge_s03_c010-full_annotation.xml'
reader = ReadData(path)
gt, num_iter = reader.getGTfromXML()
sortedFrames, sortedBBOX = reader.bboxInFrame(gt)
gtInfo = reader.joinBBOXfromFrame(sortedFrames, sortedBBOX, isGT=True)
summary = []
addNoise = True
addDrop = False
if addNoise:
stdPixels = np.linspace(0, 100, 11)
for n in range(len(stdPixels)):
std = int(stdPixels[n])
predictedBBOX = []
for i in range(len(sortedBBOX)):
noisyBBOX = sortedBBOX[i] + np.random.normal(0, std, 4) # mean 0 , std = stdPixels
predictedBBOX.append(noisyBBOX)
predictionsInfo = reader.joinBBOXfromFrame(sortedFrames, predictedBBOX, isGT=False)
rec, prec, ap, meanIoU = ap_score(gtInfo, predictionsInfo, num_bboxes=len(sortedFrames), ovthresh=0.5)
print('Noise std:', std)
print('mAP:', np.mean(ap))
print('Mean IOU:', meanIoU)
gtInfo = reader.resetGT(gtInfo)
summary.append({"std": std, "prec": np.mean(prec), "rec": np.mean(rec), "iou":meanIoU,"mAP":np.mean(ap)})
if addDrop:
dropThr = np.linspace(0, 0.9, 11)
for n in range(len(dropThr)):
predictedBBOX = []
for i in range(len(sortedBBOX)):
drop_box = random.rand()<dropThr[n]
if not drop_box:
predictedBBOX.append(sortedBBOX[i])
else:
predictedBBOX.append(None)
predictionsInfo = reader.joinBBOXfromFrame(sortedFrames, predictedBBOX, isGT=False)
rec, prec, ap, meanIoU = ap_score(gtInfo, predictionsInfo, num_bboxes=len(sortedFrames), ovthresh=0.5)
print('Thr:', dropThr[n])
print('mAP:', np.mean(ap))
print('Mean IOU:', meanIoU)
gtInfo = reader.resetGT(gtInfo)
summary.append(
{"dropThr": dropThr[n], "prec": np.mean(prec), "rec": np.mean(rec), "iou": meanIoU, "mAP": np.mean(ap)})
plots = True
if plots and addNoise:
graph = PlotCreator()
graph.plotCurve(datax=[dict['std'] for dict in summary],datay=[dict['iou'] for dict in summary],labelx='Noise std (pixels)',labely='IoU')
graph.plotCurve(datax=[dict['std'] for dict in summary],datay=[dict['mAP'] for dict in summary],labelx='Noise std (pixels)',labely='mAP')
elif plots and addDrop:
graph = PlotCreator()
graph.plotCurve(datax=[dict['dropThr'] for dict in summary],datay=[dict['iou'] for dict in summary],labelx='Drop Thr',labely='IoU')
graph.plotCurve(datax=[dict['dropThr'] for dict in summary],datay=[dict['mAP'] for dict in summary],labelx='Drop Thr',labely='mAP')
def task2():
# Read gt file
path = 'ai_challenge_s03_c010-full_annotation.xml'
reader = ReadData(path)
# Compute the mean and variance for each of the pixels along the 25% of the video
gaussModel = GaussianModel(
path='/home/mar/Desktop/M6/Lab1/AICity_data/train/S03/c010/vdo.avi',
color_space='gray')
vidLen = gaussModel.retVidLen()
# Load gt for plot
gt, num_iter = reader.getGTfromXML()
gt = reader.preprocessGT(gt)
sortedFrames, sortedBBOX, numBBOX = reader.bboxInFrame(
gt, int(vidLen * 0.25))
gtInfo = reader.joinBBOXfromFrame(sortedFrames, sortedBBOX, isGT=True)
# Separate foreground from background
alpha = [3, 4, 5, 6]
rho_values = [0.001, 0.01, 0.1, 0.5]
configurations = []
ap_results = []
iou_results = []
for a in alpha:
for r in rho_values:
# Compute mean and std
gaussModel.mean_std_Welford()
print('Alpha:', a)
predictionsInfo, num_bboxes = gaussModel.foreground_extraction_task2(
showVid=True, gt=gtInfo, alpha=a, rho=r, adaptive=True)
gtInfo = reader.resetGT(gtInfo)
rec, prec, ap, meanIoU = ap_score(gtInfo,
predictionsInfo,
num_bboxes=num_bboxes,
ovthresh=0.5)
configurations.append([a, r])
ap_results.append(ap)
iou_results.append(meanIoU)
print("")
print("Alpha: ", a)
print("Rho: ", r)
print('mAP:', ap)
print('Mean IoU:', meanIoU)
with open('ap_results_2.pkl', 'wb') as handle:
pickle.dump(ap_results, handle)
with open('iou_results_2".pkl', 'wb') as handle:
pickle.dump(iou_results, handle)
with open('configurations_2.pkl', 'wb') as handle:
pickle.dump(configurations, handle)
def task11_12():
# Read gt file
path = 'ai_challenge_s03_c010-full_annotation.xml'
reader = ReadData(path)
# Compute the mean and variance for each of the pixels along the 25% of the video
gaussModel = GaussianModel(
path='/home/mar/Desktop/M6/Lab1/AICity_data/train/S03/c010/vdo.avi',
color_space='gray')
vidLen = gaussModel.retVidLen()
# Load gt for plot
gt, num_iter = reader.getGTfromXML()
gt = reader.preprocessGT(gt)
sortedFrames, sortedBBOX, numBBOX = reader.bboxInFrame(
gt, int(vidLen * 0.25))
gtInfo = reader.joinBBOXfromFrame(sortedFrames, sortedBBOX, isGT=True)
# Compute mean and std
gaussModel.mean_std_Welford()
# Separate foreground from background
alpha = [6.25]
summary = []
for a in alpha:
print('Alpha:', a)
predictionsInfo, num_bboxes = gaussModel.foreground_extraction(
showVid=True, gt=gtInfo, alpha=a, noiseRemoval=True)
# --------------------------------------TASK 1.2-------------------------------------------------
gtInfo = reader.resetGT(gtInfo)
rec, prec, ap, meanIoU = ap_score(gtInfo,
predictionsInfo,
num_bboxes=num_bboxes,
ovthresh=0.5)
print('mAP:', ap)
print('Mean IoU:', meanIoU)
summary.append({"alpha": a, "iou": meanIoU, "mAP": np.mean(ap)})
plots = False
if plots:
graph = PlotCreator()
graph.plotCurve(datax=[dict['alpha'] for dict in summary],
datay=[dict['iou'] for dict in summary],
labelx='Alpha',
labely='IoU',
name='iouAlpha')
graph.plotCurve(datax=[dict['alpha'] for dict in summary],
datay=[dict['mAP'] for dict in summary],
labelx='Alpha',
labely='mAP',
name='mAPAlpha')
def get_aicity_dataset(frame_idx_list):
path = '/home/group09/code/week6/datasets/AICity_data/train/S03/c010/ai_challenge_s03_c010-full_annotation.xml'
video_path = '/home/group09/code/week6/datasets/AICity_data/train/S03/c010/vdo.avi'
reader = ReadData(path)
gt, num_iter = reader.getGTfromXML()
sortedFrames, sortedBBOX, numBBOX = reader.bboxInFrame(gt, 0, 2141)
gtInfo = reader.joinBBOXfromFrame(sortedFrames, sortedBBOX, isGT=True)
dataset_dicts = []
directory = '/home/group09/code/week6/datasets/AICity_data/AICity_frames'
for frame_idx in tqdm(frame_idx_list):
filename = str(frame_idx).zfill(4) + '.png'
record = {}
im_path = os.path.join(directory, filename)
im = cv2.imread(im_path)
print(filename)
height, width = im.shape[:2]
record["file_name"] = im_path
record["image_id"] = str(frame_idx).zfill(4)
record["height"] = height
record["width"] = width
classes = ['Car']
objs = []
for [
x1, y1, x2, y2
] in gtInfo[frame_idx]['bbox']: # for every bbox in a frame's gt
class_id = 0
obj = {
"type": 'Car',
"bbox": [x1, y1, x2, y2],
"bbox_mode": BoxMode.XYXY_ABS,
"category_id": 0
}
objs.append(obj)
record["annotations"] = objs
dataset_dicts.append(record)
return dataset_dicts
def task2():
path = 'ai_challenge_s03_c010-full_annotation.xml'
video_path = 'AICity_data/train/S03/c010/vdo.avi'
# Networks mask_rcnn, det_ssd512 and det_yolo3 (task 1.2)
pred_paths = ['AICity_data/train/S03/c010/det/det_mask_rcnn.txt',
'AICity_data/train/S03/c010/det/det_ssd512.txt',
'AICity_data/train/S03/c010/det/det_yolo3.txt']
# Import GT
reader = ReadData(path)
gt, num_iter = reader.getGTfromXML()
sortedFrames_gt, sortedBBOX_gt = reader.bboxInFrame(gt)
gtInfo = reader.joinBBOXfromFrame(sortedFrames_gt, sortedBBOX_gt, isGT=True)
# Noisy GT (task 1.1)
stdPixels = 10.0
predictedBBOX = []
for i in range(len(sortedFrames_gt)):
noisyBBOX = sortedBBOX_gt[i] + np.random.normal(0, stdPixels, 4)#mean 0 , std = stdPixels
predictedBBOX.append(noisyBBOX)
predictionsInfo = reader.joinBBOXfromFrame(sortedFrames_gt,predictedBBOX,isGT=False)
rec, prec, ap, meanIoU = ap_score(gtInfo,predictionsInfo,num_bboxes=len(sortedFrames_gt),ovthresh=0.5)
video_with_bbox(video_path,'noisyGT', gtInfo, predictionsInfo)
meanIoUvideoplot('noisyGT', meanIoU)
pred_nets = ['mask_rcnn', 'det_ssd512', 'det_yolo3']
for pc in range(len(pred_nets)):
reader = ReadData(pred_paths[pc])
pred, _ = reader.getPredfromTXT()
sortedFrames_pred, sortedBBOX_pred, sortedScore_pred = reader.bboxInFrame_Score(pred)
predictionsInfo = reader.joinBBOXfromFrame_Score(sortedFrames_pred, sortedBBOX_pred, sortedScore_pred,isGT=False)
rec, prec, ap, meanIoU = VOC_ap_score(gtInfo,predictionsInfo,num_bboxes=len(sortedFrames_gt),ovthresh=0.5)
video_with_bbox(video_path, pred_nets[pc], gtInfo, predictionsInfo)
meanIoUvideoplot(pred_nets[pc], meanIoU)
def task12():
path = 'ai_challenge_s03_c010-full_annotation.xml'
pred_paths = ['AICity_data/train/S03/c010/det/det_mask_rcnn.txt',
'AICity_data/train/S03/c010/det/det_ssd512.txt',
'AICity_data/train/S03/c010/det/det_yolo3.txt']
pred_nets = ['mask_rcnn', 'det_ssd512', 'det_yolo3']
for pc in range(len(pred_nets)):
reader = ReadData(path)
gt, num_iter = reader.getGTfromXML()
sortedFrames_gt, sortedBBOX_gt = reader.bboxInFrame(gt)
gtInfo = reader.joinBBOXfromFrame(sortedFrames_gt, sortedBBOX_gt, isGT=True)
reader = ReadData(pred_paths[pc])
pred, _ = reader.getPredfromTXT()
sortedFrames_pred, sortedBBOX_pred, sortedScore_pred = reader.bboxInFrame_Score(pred)
predictionsInfo = reader.joinBBOXfromFrame_Score(sortedFrames_pred, sortedBBOX_pred, sortedScore_pred,isGT=False)
rec, prec, ap, meanIoU = VOC_ap_score(gtInfo,predictionsInfo,num_bboxes=len(sortedFrames_gt),ovthresh=0.5)
print('Inference with',pred_nets[pc])
print('mAP:',np.mean(ap))
print('meanIoU:', np.mean(meanIoU))
def torchModel(model_name, video_path, xml_path, init_frame, end_frame):
reader = ReadData(xml_path)
gt, num_iter = reader.getGTfromXML()
vid = VideoModel(path=video_path, color_space='gray')
vidLen = vid.retVidLen()
sortedFrames, sortedBBOX, numBBOX = reader.bboxInFrame(
gt, init_frame, end_frame)
gtInfo = reader.joinBBOXfromFrame(sortedFrames, sortedBBOX, isGT=True)
if model_name == 'maskRCNN':
model = detection.maskrcnn_resnet50_fpn(pretrained=True)
vidCapture = cv2.VideoCapture(video_path)
device = torch.device(
'cuda') if torch.cuda.is_available() else torch.device('cpu')
model.to(device)
model.eval()
with torch.no_grad():
thr = [0.3]
tensor = transforms.ToTensor()
count = 0
for minConf in thr:
labels = []
boxes = []
scores = []
frames = []
for fr in tqdm(range(init_frame, end_frame)):
vidCapture.set(cv2.CAP_PROP_POS_FRAMES, fr)
im = vidCapture.read()[1]
x = [tensor(im).to(device)]
bbox_pred = model(x)[0]
ordered_bbox = list(
zip(bbox_pred['labels'], bbox_pred['scores'],
bbox_pred['boxes']))
# get car bboxes
car_bbox = []
for pred in ordered_bbox:
if pred[0] == 3 and pred[1] > minConf:
car_bbox.append(pred)
for box in car_bbox:
labels.append('car')
scores.append(box[1])
boxes.append(box[2])
frames.append(fr)
count += 1
predictionsInfo = reader.fixFormat(frames, boxes, labels, scores,
False)
gtInfo = reader.resetGT(gtInfo)
rec, prec, ap, meanIoU, meanIoUF = ap_score(gtInfo,
predictionsInfo,
num_bboxes=len(boxes),
ovthresh=0.5)
showVid = True
if showVid:
graph = PlotCreator()
graph.plotVid(init_frame, end_frame, vidCapture, gtInfo,
predictionsInfo)
return rec, prec, ap, meanIoU, meanIoUF
def detectronModels(model_name,video_path,xml_path,init_frame, end_frame):
# set up configuration
cfg = get_cfg()
model_name = 'fasterRCNN'
# Adding the configuration to a desired model
if model_name == 'retinaNet':
cfg.merge_from_file(model_zoo.get_config_file("COCO-Detection/retinanet_R_50_FPN_3x.yaml"))
cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url("COCO-Detection/retinanet_R_50_FPN_3x.yaml")
elif model_name == 'fasterRCNN':
cfg.merge_from_file(model_zoo.get_config_file("COCO-Detection/faster_rcnn_R_50_FPN_3x.yaml"))
cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url("COCO-Detection/faster_rcnn_R_50_FPN_3x.yaml")
############### INFERENCE
cap = cv2.VideoCapture(video_path)
if model_name == 'retinaNet':
cfg.MODEL.RETINANET.SCORE_THRESH_TEST = 0.3
else:
cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.3
predictor = DefaultPredictor(cfg)
frames = []
scores = []
labels = []
bboxes = []
boxesScore_pkl=[]
for frame_idx in tqdm(range(init_frame,end_frame)):
cap.set(cv2.CAP_PROP_POS_FRAMES, frame_idx)
im = cap.read()[1]
output = predictor(im)
car_instances = output["instances"][output["instances"].pred_classes==2].to("cpu")
box_pkl = []
for idx in range(len(car_instances.pred_boxes)):
bbox = car_instances.pred_boxes[idx]
score = car_instances.scores[idx]
bboxes.append(bbox.tensor[0])
frames.append(frame_idx)
scores.append(score)
labels.append('Car')
box_pkl.append([bbox.tensor[0].cpu().numpy(),score.cpu().numpy()])
boxesScore_pkl.append(box_pkl)
with open('boxesScores.pkl', 'wb') as f:
pickle.dump(boxesScore_pkl, f)
prediction_results = [frames,scores,labels,bboxes]
with open('prediction_results_retina.pkl', 'wb') as f:
pickle.dump(prediction_results, f)
reader = ReadData(xml_path)
predictionsInfo = reader.fixFormat(frames, bboxes, labels, scores, False)
gt, num_iter = reader.getGTfromXML()
sortedFrames, sortedBBOX, numBBOX = reader.bboxInFrame(gt=gt,initFrame=init_frame,endFrame=end_frame-1)
gtInfo = reader.joinBBOXfromFrame(sortedFrames, sortedBBOX, isGT=True)
gtInfo = reader.resetGT(gtInfo)
rec, prec, ap, meanIoU,meanIoUF = ap_score(gtInfo, predictionsInfo, num_bboxes=len(bboxes), ovthresh=0.5)
showVid = True
if showVid:
graph = PlotCreator()
graph.plotVid(init_frame, end_frame, cap, gtInfo, predictionsInfo)
return rec,prec,ap,meanIoU,meanIoUF