def week2_adaptive_hsv(video: Video, debug=False) -> Iterator[Frame]:
model_mean, model_std = get_background_model(video,
int(2141 * 0.25),
total_frames=int(2141 * 0.25),
pixel_value=PixelValue.HSV)
ground_truth = read_detections(
'../datasets/AICity_data/train/S03/c010/gt/gt.txt')
frame_id = int(2141 * 0.25)
roi = cv2.cvtColor(
cv2.imread('../datasets/AICity_data/train/S03/c010/roi.jpg'),
cv2.COLOR_BGR2GRAY)
for im, mask in gaussian_model_adaptive(video,
int(2141 * 0.25),
model_mean,
model_std,
total_frames=int(2141 * 0.10),
pixel_value=PixelValue.HSV,
alpha=1.75,
rho=0.01):
mask = mask & roi
if debug:
cv2.imshow('f', mask)
cv2.waitKey()
mask = opening(mask, 7)
if debug:
cv2.imshow('f', mask)
cv2.waitKey()
mask = closing(mask, 35)
if debug:
cv2.imshow('f', mask)
cv2.waitKey()
mask, detections = find_boxes(mask)
frame = Frame(frame_id)
frame.detections = detections
frame.ground_truth = ground_truth[frame_id]
if debug:
mask2 = cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR)
for detection in detections:
cv2.rectangle(
mask2,
(int(detection.top_left[1]), int(detection.top_left[0])),
(int(detection.get_bottom_right()[1]),
int(detection.get_bottom_right()[0])), (0, 255, 0), 5)
for gt in ground_truth[frame_id]:
cv2.rectangle(mask2,
(int(gt.top_left[1]), int(gt.top_left[0])),
(int(gt.get_bottom_right()[1]),
int(gt.get_bottom_right()[0])), (255, 0, 0), 5)
cv2.imshow('f', mask2)
cv2.waitKey()
yield im, mask, frame
frame_id += 1
def compute_map_gt_det(det_file, gt_file):
det_list = read_detections(det_file)
gt_list = read_detections(gt_file)
frames = []
for i in range(0, len(det_list)):
frame = Frame(i)
frame.detections = det_list[i]
frame.ground_truth = gt_list[i]
frames.append(frame)
mAP = mean_average_precision(frames, ignore_classes=True)
return mAP
def main():
start_frame = 1440
end_frame = 1789
gt = read_annotations('../annotations', start_frame, end_frame)
alg = 'mask_rcnn'
detections = read_detections('../datasets/AICity_data/train/S03/c010/det/det_{0}.txt'.format(alg))
kalman = KalmanTracking()
for i in range(start_frame, end_frame):
f = Frame(i)
f.detections = detections[i]
f.ground_truth = gt[i - start_frame]
kalman(f)
print(seq(f.detections).map(lambda d: d.id).to_list())
def main():
im_1440 = cv2.imread(
"../datasets/AICity_data_S03_c010_1440/frame_1440.jpg")
top_left = [995, 410]
width = 1241 - 995
height = 605 - 410
ground_truth = [Detection('', 'car', top_left, width, height)]
"""
DETECTIONS FROM ALTERED GROUND TRUTH
"""
frame = Frame(0, ground_truth)
frame.detections = alter_detections(ground_truth)
plot_frame(im_1440, frame)
iou = frame.get_detection_iou()
iou_mean = frame.get_detection_iou_mean()
print("IOU: ", iou, "IOU mean", iou_mean)
def addframe(frame_name,hdr,obsdate,obstime,filename,path,instrument=""):
""" addframe(frame_name,hdr,obsdate,obstime,filename,path,instrument)
frame_name - the unique frame name
hdr - FITS header
obsdate- datetime object that is the date from hdr
obstime- datetime object that is the time from hdr
filename - the filename on the file system, this should be related to the
frame_name
path - the relative path to the object, not including what is the datapath
This builds the actual frame instance.
frame_name is a unique key for the name of the datafile. Based on the hdr
data, the routine fills in the information that the db needs to classify
the file and run the pipelines with the correct input
"""
msg = ''
frame = Frame(frame_name,path,
filename,
header = hdr,
use_mark = True,
observeddate = obsdate,
observedtime = datetime.combine(obsdate,obstime))
if instrument == "":
instrument = find_instrument_name(hdr)
# print "Instrument = ",instrument
if instrument == "lrisred" :
# print "Is a red frame!"
frame.instrument = Instrument(name=u'lrisred',display_name=u'LRISRED'
)
frame = LRISFrameutil.addframelris(frame,hdr,instrument)
elif instrument == "lrisblue":
frame.instrument = Instrument(name=u'lrisblue',display_name=u'LRISBLUE',
)
frame = LRISFrameutil.addframelris(frame,hdr,instrument)
if not frame:
msg="There is something wrong with %s" % frame_name
return(msg,frame)
def query(self, picture: Picture, frame: Frame = None) -> List[Picture]:
im = picture.get_image()
if frame and frame.is_valid():
side = int(math.sqrt(frame.get_area()) * 0.8)
m = frame.get_perspective_matrix(
np.array([[0, side - 1], [side - 1, side - 1], [side - 1, 0],
[0, 0]]))
im = cv2.warpPerspective(im, m, (side, side))
# plt.imshow(cv2.cvtColor(im, cv2.COLOR_BGR2RGB))
# plt.show()
kp, des = self.orb.detectAndCompute(im, None)
return (
seq(self.db).map(lambda p: (p[0], self.bf.match(p[2], des))).
map(lambda p: (p[0], seq(p[1]).filter(lambda d: d.distance < max(
THRESHOLD,
seq(p[1]).map(lambda m: m.distance).min())).to_list())).
map(lambda p: (p[0], len(p[1]))).filter(lambda p: p[1] > 4).sorted(
lambda p: p[1],
reverse=True).map(lambda p: p[0]).take(10).to_list())
def week2_soa(video: Video, debug=False) -> Iterator[Frame]:
th = 150
frame_id = 0
fgbg = cv.createBackgroundSubtractorMOG2()
ground_truth = read_detections(
'../datasets/AICity_data/train/S03/c010/gt/gt.txt')
roi = cv.cvtColor(
cv.imread('../datasets/AICity_data/train/S03/c010/roi.jpg'),
cv.COLOR_BGR2GRAY)
for im in tqdm(video.get_frames(),
total=2141,
file=sys.stdout,
desc='Training model...'):
mask = fgbg.apply(im)
mask[mask < th] = 0
mask.astype(np.uint8) * 255
mask = mask & roi
mask = opening(mask, 5)
# cv.imshow('f', mask)
# cv.waitKey()
mask = closing(mask, 25)
# cv.imshow('f', mask)
# cv.waitKey()
mask, detections = find_boxes(mask)
frame = Frame(frame_id)
frame.detections = detections
frame.ground_truth = ground_truth[frame_id]
frame_id += 1
yield im, mask, frame
def makecallist(callist,caldir,stddir):
try:
calfile = open(callist)
except:
return(False)
calfilestr = calfile.read()
calfilelist = calfilestr.splitlines()
calframes = []
flags = ""
for line in calfilelist:
linearray = line.split()
if len(linearray) == 7:
[filename,camera,filetype,grating,wavelen,xb,yb] = linearray
elif len(linearray) > 7:
[filename,camera,filetype,grating,wavelen,xb,yb] = linearray[0:6]
flags = " ".join(linearray[7:])
else:
return([])
calframepath = findframe(caldir,camera,grating,filename)
if calframepath:
calframe = Frame(name=filename,path=calframepath,display_name=filename)
calframe.grating = grating
calframe.wavelength = float(wavelen)
calframe.type = filetype
calframe.xbinning = int(xb)
calframe.ybinning = int(yb)
calframe.flags = flags
if camera == "r":
calframe.instrument=Instrument(name=u'lrisred',display_name=u'LRISRED')
elif camera == "b":
calframe.instrument =Instrument(name=u'lrisblue',display_name=u'LRISBLUE')
calframes.append(calframe)
make_gratingdir(filename,grating,stddir)
else:
print "Cannot find file %s in %s or appropriate subdirectories." % (filename,caldir)
return(calframes)
def get_frames(self) -> Iterable[Frame]:
video = cv2.VideoCapture(os.path.join(self.video_path, "vdo.avi"))
full_detections = read_detections(
os.path.join(self.video_path, "det/det_yolo3.txt"))
if os.path.exists(os.path.join(self.video_path, "gt/gt.txt")):
full_ground_truth = read_detections(
os.path.join(self.video_path, "gt/gt.txt"))
else:
full_ground_truth = []
count = 0
while video.isOpened():
ret, frame = video.read()
if ret:
det = full_detections[count] if count < len(
full_detections) else []
gt = full_ground_truth[count] if count < len(
full_ground_truth) else []
yield Frame(count, det, gt, frame)
count += 1
else:
video.release()
def createFrame(self,timeNow):
if self.frame :
print('Frame already created')
else :
self.frame = Frame.Frame(timeNow,self)
sio = socketio.Server()
app = Flask(__name__)
model = None
prev_image_array = None
# Let to access the command line arguments globally
args = None
# Integration buffer for the PI speed controller
ibuf=0
# CNN input data shape. It is set during initialization
input_size = []
# Create a single Frame object that we will use for image preprocessing
frame = Frame()
@sio.on('telemetry')
def telemetry(sid, data):
global ibuf
if data:
# The current steering angle of the car
steering_angle = data["steering_angle"]
# The current throttle of the car
throttle = data["throttle"]
# The current speed of the car
speed = data["speed"]
# The current image from the center camera of the car
imgString = data["image"]
image_src = Image.open(BytesIO(base64.b64decode(imgString)))
def make_callist(callist,caldir):
"""make_callist(callist, caldir)
This reads in the file specified as callist.
The path must be absolute, or the routine will return False
The caldir is used to constructed the Frame object, so that future
functions will know where the actual is located (thus, in theory,
the calibration file and the callist can be in two different
places.)
"""
try:
calfile = open(callist)
except:
return(False)
calfilestr = calfile.read()
calfilelist = calfilestr.splitlines()
calframes = []
flags = ""
for line in calfilelist:
linearray = line.split()
if len(linearray) == 7:
[filename,camera,filetype,grating,wavelen,xb,yb] = linearray
elif len(linearray) > 7:
[filename,camera,filetype,grating,wavelen,xb,yb] = linearray[0:6]
flags = " ".join(linearray[6:])
else:
return([])
calframe = Frame(name=filename,path=caldir,display_name=filename)
calframe.grating = grating
calframe.wavelength = wavelen
calframe.type = filetype
calframe.xbinning = xb
calframe.ybinning = yb
calframe.flags = flags
calframe.exptime = 1
calframe.target = 'horizon lock'
calframe.aperture = 'long_1.0'
if camera == "r":
calframe.instrument=Instrument(name=u'lrisred',display_name=u'LRISRED')
elif camera == "b":
calframe.instrument =Instrument(name=u'lrisblue',display_name=u'LRISBLUE')
calframes.append(calframe)
# make_gratingdir(filename,grating,stddir)
return(calframes)
def off_the_shelf_yolo(tracking, debug=False, *args, **kwargs):
video = Video("../datasets/AICity_data/train/S03/c010/frames")
detection_transform = DetectionTransform()
classes = utils.load_classes('../config/coco.names')
gt = read_annotations(
'../datasets/AICity_data/train/S03/c010/m6-full_annotation.xml')
model = Darknet('../config/yolov3.cfg')
model.load_weights('../weights/fine_tuned_yolo_freeze.weights')
if torch.cuda.is_available():
model = model.cuda()
frames = []
last_im = None
model.eval()
with torch.no_grad():
for i, im in tqdm(enumerate(video.get_frames(start=len(video) // 4)),
total=len(video),
file=sys.stdout,
desc='Yolo'):
im_tensor = detection_transform(im)
im_tensor = im_tensor.view((-1, ) + im_tensor.size())
if torch.cuda.is_available():
im_tensor = im_tensor.cuda()
detections = model.forward(im_tensor)
detections = utils.non_max_suppression(detections,
80,
conf_thres=.6,
nms_thres=0.3)
frame = Frame(i + (len(video) // 4))
frame.ground_truth = gt[frame.id]
for d in detections[0]:
if int(d[6]) in VALID_LABELS:
bbox = d.cpu().numpy()
det = Detection(-1,
classes[int(d[6])], (bbox[0], bbox[1]),
width=bbox[2] - bbox[0],
height=bbox[3] - bbox[1],
confidence=d[5])
detection_transform.unshrink_detection(det)
frame.detections.append(det)
if tracking is not None:
last_frame = None if len(frames) == 0 else frames[-1]
tracking(frame=frame,
im=im,
last_frame=last_frame,
last_im=last_im,
frames=frames,
debug=False)
frames.append(frame)
last_im = im
if debug:
plt.figure()
for det in frame.detections:
rect = patches.Rectangle(det.top_left,
det.width,
det.height,
linewidth=2,
edgecolor='blue',
facecolor='none')
plt.gca().add_patch(rect)
if tracking is None:
text = '{}'.format(det.label)
else:
text = '{} ~ {}'.format(det.label, det.id)
plt.text(det.top_left[0],
det.top_left[1],
s=text,
color='white',
verticalalignment='top',
bbox={
'color': 'blue',
'pad': 0
})
plt.imshow(im)
plt.axis('off')
# plt.savefig('../video/video_yolo_fine_tune_good/frame_{:04d}'.format(i))
plt.show()
plt.close()
# iou_over_time(frames)
mAP = mean_average_precision(frames)
print("YOLO mAP:", mAP)
def off_the_shelf_ssd(tracking, debug=False, **kwargs):
if cuda.is_available():
torch.set_default_tensor_type('torch.cuda.FloatTensor')
gt = read_annotations(
'../datasets/AICity_data/train/S03/c010/m6-full_annotation.xml')
video = Video("../datasets/AICity_data/train/S03/c010/frames")
trans = transforms.Compose(
[transforms.Resize((300, 300)),
transforms.ToTensor()])
labels = ( # always index 0
'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat',
'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person',
'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor')
model = build_ssd('test', 300, 21) # initialize SSD
model.load_weights('../weights/ssd300_mAP_77.43_v2.pth')
if torch.cuda.is_available():
model = model.cuda()
frames = []
model.eval()
with torch.no_grad():
for i, im in enumerate(video.get_frames()):
im_tensor = trans(im)
im_tensor = im_tensor.view((-1, ) + im_tensor.size())
if torch.cuda.is_available():
im_tensor = im_tensor.cuda()
output = model.forward(im_tensor)
detections = output.data
w = im.width
h = im.height
frame = Frame(i)
frame.ground_truth = gt[frame.id]
# skip j = 0, because it's the background class
for j in (2, 6, 7, 14):
dets = detections[0, j, :]
mask = dets[:, 0].gt(0.).expand(5, dets.size(0)).t()
dets = torch.masked_select(dets, mask).view(-1, 5)
if dets.size(0) == 0:
continue
boxes = dets[:, 1:]
scores = dets[:, 0].cpu().numpy()
cls_dets = np.hstack((boxes.cpu().numpy(),
scores[:,
np.newaxis])).astype(np.float32,
copy=False)
for cls_det in cls_dets:
x1 = int(w * cls_det[0])
y1 = int(h * cls_det[1])
det = Detection(-1,
labels[j - 1], (x1, y1),
width=w * (cls_det[2] - cls_det[0]),
height=h * (cls_det[3] - cls_det[1]),
confidence=cls_det[4])
frame.detections.append(det)
# kalman(frame)
if tracking is not None:
tracking(frame, frames, debug=debug)
frames.append(frame)
if debug:
plt.figure()
for det in frame.detections:
rect = patches.Rectangle(det.top_left,
det.width,
det.height,
linewidth=2,
edgecolor='blue',
facecolor='none')
plt.gca().add_patch(rect)
plt.text(det.top_left[0],
det.top_left[1],
s='{} ~ {}'.format(det.label, det.id),
color='white',
verticalalignment='top',
bbox={
'color': 'blue',
'pad': 0
})
plt.imshow(im)
plt.axis('off')
# plt.savefig('../video/video_ssd_KalmanID/frame_{:04d}'.format(i))
plt.show()
plt.close()
#iou_over_time(frames)
mAP = mean_average_precision(frames)
print("SSD mAP:", mAP)
def query(self, picture: Picture) -> (List[Picture], Frame):
res = self.compare_histograms.query(picture)
return res, Frame()
def get_frame_with_lines(im: np.ndarray) -> Frame:
scale = min(MAX_SIDE / im.shape[0], MAX_SIDE / im.shape[1])
resized = cv2.resize(im, (0, 0), fx=scale, fy=scale)
if SHOW_OUTPUT:
plt.imshow(cv2.cvtColor(resized, cv2.COLOR_BGR2RGB))
plt.show()
gray = cv2.cvtColor(resized, cv2.COLOR_BGR2GRAY)
if SHOW_OUTPUT:
plt.imshow(gray, 'gray')
plt.show()
gray = cv2.GaussianBlur(gray, ksize=(5, 5), sigmaX=0)
# uses the above two partial derivatives
sobelx = cv2.Sobel(gray, cv2.CV_16S, 1, 0, ksize=3)
sobely = cv2.Sobel(gray, cv2.CV_16S, 0, 1, ksize=3)
abs_gradientx = cv2.convertScaleAbs(sobelx)
abs_gradienty = cv2.convertScaleAbs(sobely)
# combine the two in equal proportions
gray = cv2.addWeighted(abs_gradientx, 0.5, abs_gradienty, 0.5, 0)
gray = cv2.GaussianBlur(gray, ksize=(5, 5), sigmaX=0)
gray = cv2.Canny(gray, threshold1=0, threshold2=50, apertureSize=3)
if SHOW_OUTPUT:
plt.imshow(gray, 'gray')
plt.show()
lines = cv2.HoughLinesP(gray, rho=1, theta=np.pi / 180, threshold=80, minLineLength=100, maxLineGap=10)
imres = None
if SHOW_OUTPUT:
im2 = resized.copy()
for line in lines:
cv2.line(im2, (line[0][0], line[0][1]), (line[0][2], line[0][3]), (0, 0, 255), 3)
plt.imshow(cv2.cvtColor(im2, cv2.COLOR_BGR2RGB))
plt.show()
imres = cv2.cvtColor(resized, cv2.COLOR_BGR2RGB)
intersections = get_intersections(lines)
if SHOW_OUTPUT:
im2 = resized.copy()
for p in intersections:
cv2.circle(im2, (int(p[0]), int(p[1])), 3, (0, 0, 255), -1)
plt.imshow(cv2.cvtColor(im2, cv2.COLOR_BGR2RGB))
plt.show()
intersections = simplify_intersections(intersections)
if SHOW_OUTPUT:
im2 = resized.copy()
for p in intersections:
cv2.circle(im2, (int(p[0]), int(p[1])), 3, (0, 0, 255), -1)
plt.imshow(cv2.cvtColor(im2, cv2.COLOR_BGR2RGB))
plt.show()
points = [(0., 0.), (0., 0.), (0., 0.), (0., 0.)]
angle = 0
if len(intersections) > 4:
points, angle = magic(intersections, im.shape[0] * scale, im.shape[1] * scale)
if SHOW_OUTPUT:
for p in points:
cv2.circle(imres, (int(p[0]), int(p[1])), 3, (255, 0, 0), thickness=-1)
if SHOW_OUTPUT:
plt.imshow(imres)
plt.show()
# Undo scale
points = (seq(points)
.map(lambda point: (int(point[0] / scale), int(point[1] / scale)))
.to_list())
return Frame(points, angle)
