def optical_flow_dense(files):
# Perform Godec first on all frames
M, LS, L, S, width, height = bs_godec(files)
first_frame = get_frame(files[0])
# frames to be compared is after godec and postprocessing
godec_frame, probability = get_godec_frame(M, L, S, width, height, 0)
img, centroids = postprocess_img(godec_frame, all_images=False)
prev_gray = img
ims = init_comparison_plot(first_frame,
["Original", "Thresholded", "FlowS"], 1, 3)
test = cv.cvtColor(first_frame.astype("uint8"), cv.COLOR_GRAY2BGR)
hsv_mask = np.zeros_like(test)
hsv_mask[..., 1] = 255
window_name = "Dense Optical Flow"
counter = 1
while counter < len(files):
print(counter)
godec_frame, probability = get_godec_frame(M, L, S, width, height,
counter)
img, centroids = postprocess_img(godec_frame, all_images=False)
next_gray = img
flow = cv.calcOpticalFlowFarneback(prev_gray,
next_gray,
None,
pyr_scale=0.5,
levels=5,
winsize=11,
iterations=5,
poly_n=5,
poly_sigma=1.1,
flags=0)
magnitude, angle = cv.cartToPolar(flow[..., 0], flow[..., 1])
hsv_mask[
...,
0] = angle * 180 / np.pi / 2 # Set image hue according to the optical flow direction
hsv_mask[..., 2] = cv.normalize(
magnitude, None, 0, 255, cv.NORM_MINMAX
) # Set image value according to the optical flow magnitude (normalized)
# plotting of grayscale flowmap and data heatmap
update_comparison_plot(
ims, [get_frame(files[counter]), next_gray, hsv_mask])
plt.title("Max Magnitude :" + str(np.amax(magnitude)) +
"\nMax Angle:" + str(np.amax(angle)))
create_folder_if_absent("optical_flow_pics")
plt.savefig("optical_flow_pics/{}.png".format(counter))
prev_gray = next_gray
k = cv.waitKey(30) & 0xff
counter += 1
def test_foreground_probability(savegif=False):
subplt_titles = ["Original", "Foreground %"]
ims = init_comparison_plot(get_frame_GREY(data[0]),
subplt_titles,
1,
2,
title="Probabilistic Foreground Detection")
for i in range(len(data)):
frame = get_frame(data[i])
L_frame = L[:, i].reshape(width, height).T
S_frame = S[:, i].reshape(width, height).T
L_probability = foreground_probability(L_frame, frame)
# this condition is now abstracted as the function cleaned_godec_img() in background_subtractio.py
S_probability = foreground_probability(S_frame, frame)
if np.amax(L_probability) > np.amax(S_probability):
probability = L_probability
img = L_frame
else:
probability = S_probability
img = S_frame
images = [normalize_frame(img), normalize_frame(probability)]
update_comparison_plot(ims, images)
create_folder_if_absent("testpics")
plt.savefig("testpics/{}.png".format(i))
if savegif:
files = get_all_files("testpics")
write_gif_from_pics(files,
"probabilistic_foreground_detection.gif",
fps=20)
def bs_pipeline(files, debug=False, save=False):
"""Background Subtraction Pipeline process
1. perform godec background subtraction
2. thresholding to make actual pixels representing the person to be more salient in the frame
3. Contour detection to detect centroid of person
4. naive detection / optical flow.
Arguments:
files {[str]} -- Array obtained from get_all_files(data_path)
Keyword Arguments:
debug {bool} -- [description] (default: {False})
"""
M, LS, L, S, width, height = bs_godec(files)
if debug:
subplt_titles = ["Original", "After Godec", "Blurred", " Thresholded", "Annotated"]
ims = init_comparison_plot(get_frame_GREY(files[0]), subplt_titles, 1, 5, title="Post Processing")
for i in tqdm(range(len(files))):
L_frame = normalize_frame(L[:, i].reshape(width, height).T)
S_frame = normalize_frame(S[:, i].reshape(width, height).T)
img = cleaned_godec_img(L_frame, S_frame, get_frame(files[i]))
images, centriods = postprocess_img(img)
images.insert(0, get_frame_GREY(files[i]))
update_comparison_plot(ims, images)
plt.savefig(bg_subtraction_pics_path+"{}.png".format(i))
def test_frame_filter(savegif=False):
noise_remover = FrameKalmanFilter()
save_path = kalman_pics_path + basename(data_path)
create_folder_if_absent(save_path)
data_i = get_frame(data[0])
subplt_titles = ["Original Image", "noise removed"]
ims, axs, fig = init_noise_reduction_plot(data_i, subplt_titles)
for i in tqdm(range(len(data))):
data_i = get_frame(data[i])
processed = noise_remover.process_frame(data_i)
update_noise_reduction_plot(ims, [data_i, processed])
plt.savefig(save_path + "/{}.png".format(i)) #make png
if savegif:
gif_pics = [save_path+ "/{}.png".format(i) for i in range(len(data))]
gif_path = kalman_gifs_path+basename(data_path)+".gif"
write_gif(gif_pics, gif_path, 0, len(gif_pics), fps=30)
optimize_size(gif_path)
def test_pixel_filter():
noise_remover = PixelKalmanFilter()
temp_over_time = [get_frame(data[i])[0][0] for i in range(len(data))]
temp_over_time_2 = copy.copy(temp_over_time)
filtered = [np.array(noise_remover.filter(temp_over_time[i]))[0][0] for i in range(len(data))]
plt.plot(np.arange(0,255,1), temp_over_time_2, "r--", np.arange(0,255,1), filtered, "bs")
plt.show()
def test_frame_filter_by_specific_pixel():
temp_over_time = []
noise_remover = FrameKalmanFilter()
for i in tqdm(range(len(data))):
processed = noise_remover.process_frame(get_frame(data[i]))
temp_over_time.append(copy.copy(processed)) # ADD THE COPY.COPY(). IF YOU DONT,IT APPENDS THE REFERENCE SO YOUR WHOLE ARRAY IS THE SAME
pixel_over_time = []
for i in tqdm(range(len(data))):
frame = get_frame(data[i]) # measurement data
pixel_over_time.append(frame[20][20]) # adds the values of a specific pixel over time to pixel_over_time
filtered_over_time = []
for i in tqdm(range(len(data))):
frame = temp_over_time[i] # filtered data
filtered_over_time.append(frame[20][20]) # adds the values of a specific pixel over time to filtered_over_time
plt.plot(np.arange(0,255,1), pixel_over_time, "r--", np.arange(0,255,1), filtered_over_time, "bs")
plt.show()
def optical_flow_lk(files, track_length=10, detect_interval=5):
print("Performing Lucas-Kanade Optical Flow")
plot = get_init_heatmap_plot()
# params for ShiTomasi corner detection
feature_params = dict(maxCorners=4,
qualityLevel=0.2,
minDistance=6,
blockSize=4)
# Parameters for lucas kanade optical flow
lk_params = dict(winSize=(3, 3),
maxLevel=3,
criteria=(cv.TERM_CRITERIA_EPS | cv.TERM_CRITERIA_COUNT,
10, 0.03))
# Take first frame and find corners in it
first_frame_gray = get_frame_GREY(files[0])
# TODO: instead of using good features to track, possibly just use contour points directly
prevPts = cv.goodFeaturesToTrack(first_frame_gray,
mask=None,
**feature_params)
color = np.random.randint(0, 255, (100, 3))
counter = 1
prevImg = first_frame_gray
while counter < len(files):
frame = get_frame_GREY(files[counter])
nextImg = frame.copy()
update_heatmap(get_frame(files[counter]), plot)
nextPts, status, err = cv.calcOpticalFlowPyrLK(prevImg, nextImg,
prevPts, None,
**lk_params)
displacement = nextPts - prevPts
if (abs(displacement) > 3).any():
print(displacement)
plt.xlabel("Displacement: {}".format(displacement))
else:
plt.xlabel("Displacement in x/y lower than 3 ")
if nextPts is None:
print("Target not moving")
prevPts = cv.goodFeaturesToTrack(frame,
mask=None,
**feature_params)
nextPts, status, err = cv.calcOpticalFlowPyrLK(
prevImg, nextImg, prevPts, None, **lk_params)
# Select good points
# each element of the vector is set to 1 if the flow for the corresponding features has been found, otherwise, it is set to 0.
good_new = nextPts[status == 1]
good_old = prevPts[status == 1]
# Now update the previous frame and previous points
prevImg = nextImg.copy()
prevPts = good_new.reshape(-1, 1, 2)
counter += 1
def test_plot(files):
array_shape = (24, 32)
min_value = 30
max_value = 40
plot = init_heatmap("Heatmap",
array_shape,
min_value,
max_value,
debug=True)
for f in files:
frame = get_frame(f)
update_heatmap(frame, plot)
def test_bs_godec_trained_noise(noise_path, gif_name, preview):
"""Test bs_godec Implementation with trained noise
Keyword Arguments:
noise_path {string}
gif_name {string}
preview {boolean}
---
This test shows you how to run the bs_godec method.
"""
N = get_frame(noise_path)
M, R = bs_godec_trained(files, N)
plot_bs_results(M, N, R, bs_pics_path, preview=preview)
def test_plot_without_labels(files):
array_shape = (24, 32)
min_value = 25
max_value = 40
plot = init_heatmap("Heatmap",
array_shape,
min_value,
max_value,
debug=False)
for i in range(len(files)):
frame = get_frame(files[i])
update_heatmap(frame, plot)
create_folder_if_absent("testpics")
plt.savefig("testpics/{}.png".format(i))
def test_godec_over_multiple_iterations(frames_per_iterations=30):
ims = init_comparison_plot(get_frame_GREY(files[0]),
subplt_titles=[
"Original", "L_frame", "S_Frame",
"Cleaned_Frame", "L_fram_%", "S_frame_%"
],
num_rows=2,
num_columns=3)
for j in range(0, len(files), frames_per_iterations):
if j + frames_per_iterations < len(files):
end_index = j + frames_per_iterations
else:
end_index = len(files)
M, LS, L, S, width, height = bs_godec(files[j:end_index],
normalize=False)
for i in range(i, end_index):
img = get_frame_GREY(files[i])
L_frame = normalize_frame(L[:, i].reshape(width, height).T)
S_frame = normalize_frame(S[:, i].reshape(width, height).T)
L_probability = foreground_probability(
L[:, i].reshape(width, height).T, get_frame(files[i]))
S_probability = foreground_probability(
S[:, i].reshape(width, height).T, get_frame(files[i]))
print("L_probability")
print(L_probability)
print("S_probability")
print(S_probability)
cleaned_frame, prob = cleaned_godec_img(L_frame, S_frame,
get_frame(files[i]))
update_comparison_plot(ims, [
img, L_frame, S_frame, cleaned_frame,
normalize_frame(L_probability),
normalize_frame(S_probability)
])
create_folder_if_absent("testpics")
plt.savefig("testpics/" + "{}.png".format(i))
def write_gif_from_npy(files, name, start=0, end=0, fps=1):
print("Plotting from {} numpy files and writing gif of {}...".format(
len(files), fps))
plot = init_heatmap(name, show=False)
end = end or len(files)
with imageio.get_writer(name, mode='I', fps=fps) as writer:
for i in tqdm(range(start, end)):
f = get_frame(files[i])
update_heatmap(f, plot)
pic_name = "pics/" + files[i] + ".png"
plt.savefig(pic_name)
writer.append_data(imageio.imread(pic_name))
writer.close()
print("Finished writing gif at {}.".format(name))
optimize_size(name)
def analyze_by_period(files, num_frames=60 * 30):
time_person_spent_in_areas = defaultdict(int)
counter = 0
while counter < num_frames:
frame = get_frame(files[counter])
areas_person_is_in = naive_binary_likelihood_by_frame(frame)
for area in areas_person_is_in:
if areas_person_is_in[area]:
time_person_spent_in_areas[area] += 1
counter += 1
total_time_spent_in_room = sum(time_person_spent_in_areas.values())
time_person_spent_in_areas = {
i: time_person_spent_in_areas[i] / total_time_spent_in_room
for i in range(8)
}
return time_person_spent_in_areas
def naive_detection_from_files(data_path, startIndex=None, endIndex=None):
heatmap_plot = get_init_heatmap_plot()
likelihood_plot = get_init_likelihood_plot()
files = get_all_files(data_path)
if startIndex == None:
startIndex = 0
if endIndex == None:
endIndex = len(files)
print(startIndex, endIndex)
for i in range(startIndex, endIndex):
frame = get_frame(files[i])
areas_person_is_in = naive_detection_by_frame(frame)
likelihood_array = [[
areas_person_is_in[i]["likelihood"] for i in range(4)
], [areas_person_is_in[i]["likelihood"] for i in range(4, 8)]]
# if debugging with plot view
update_heatmap(frame, heatmap_plot)
update_heatmap(likelihood_array, likelihood_plot)
def create_godec_input(files, normalize=True, rgb=False):
i = 0
M = None
frame = None
for f in files:
if type(f) == str:
if rgb:
frame = get_frame_RGB(f)
elif normalize:
frame = get_frame_GREY(f)
else:
frame = get_frame(f)
else:
frame = files[i]
# Stack frames as column vectors
F = frame.T.reshape(-1)
if i == 0:
M = array([F]).T
else:
M = column_stack((M, F))
i+=1
return M, frame
def get_centroid_area_history(files, debug=True, key_format="simple"):
"""
Primary function to be called for obtaining history in the following format:
{
0: {
0: ...,
1: ...,
},
1: { ... }
}
Arguments:
files {[str]} -- up to 30 mins of files, since we decided that recalibration of godec should be done every 30 mins
"""
annotated_images = []
centroid_history = []
M, LS, L, S, width, height = bs_godec(files)
for i in range(len(files)):
img = get_frame_GREY(files[i])
L_frame = normalize_frame(L[:, i].reshape(width, height).T)
S_frame = normalize_frame(S[:, i].reshape(width, height).T)
img = cleaned_godec_img(L_frame, S_frame, get_frame(files[i]))
images, centroids = postprocess_img(img)
annotated_img = images[-1]
annotated_images.append(annotated_img)
append_centroid_history(centroids, i, centroid_history)
interpolated_centroid_history = Interpolator(centroid_history).history
centroid_area_array = [
get_centroid_area_number(cnt) for cnt in interpolated_centroid_history
]
area_counter = Counter(centroid_area_array)
for i in range(8):
if i not in area_counter:
area_counter[i] = 0
if key_format == "simple":
area_movement_counter = Counter()
for i in range(len(centroid_area_array) - 1):
from_area = str(centroid_area_array[i])
to_area = str(centroid_area_array[i + 1])
label = from_area + "→" + to_area
area_movement_counter[label] += 1
elif key_format == "from_to":
area_movement_counter = {"None": Counter()}
for i in range(8):
area_movement_counter[str(i)] = Counter()
for i in range(len(centroid_area_array) - 1):
from_area = str(centroid_area_array[i])
to_area = str(centroid_area_array[i + 1])
area_movement_counter[from_area][to_area] += 1
if debug:
return area_counter, area_movement_counter, centroid_area_array, annotated_images
return area_movement_counter