def update_mhi(img, dst, diff_threshold):
global last
global mhi
global storage
global mask
global orient
global segmask
timestamp = time.clock() / CLOCKS_PER_SEC # get current time in seconds
size = cv.GetSize(img) # get current frame size
idx1 = last
if not mhi or cv.GetSize(mhi) != size:
for i in range(N):
buf[i] = cv.CreateImage(size, cv.IPL_DEPTH_8U, 1)
cv.Zero(buf[i])
mhi = cv.CreateImage(size,cv. IPL_DEPTH_32F, 1)
cv.Zero(mhi) # clear MHI at the beginning
orient = cv.CreateImage(size,cv. IPL_DEPTH_32F, 1)
segmask = cv.CreateImage(size,cv. IPL_DEPTH_32F, 1)
mask = cv.CreateImage(size,cv. IPL_DEPTH_8U, 1)
cv.CvtColor(img, buf[last], cv.CV_BGR2GRAY) # convert frame to grayscale
idx2 = (last + 1) % N # index of (last - (N-1))th frame
last = idx2
silh = buf[idx2]
cv.AbsDiff(buf[idx1], buf[idx2], silh) # get difference between frames
cv.Threshold(silh, silh, diff_threshold, 1, cv.CV_THRESH_BINARY) # and threshold it
cv.UpdateMotionHistory(silh, mhi, timestamp, MHI_DURATION) # update MHI
cv.CvtScale(mhi, mask, 255./MHI_DURATION,
(MHI_DURATION - timestamp)*255./MHI_DURATION)
cv.Zero(dst)
cv.Merge(mask, None, None, None, dst)
cv.CalcMotionGradient(mhi, mask, orient, MAX_TIME_DELTA, MIN_TIME_DELTA, 3)
if not storage:
storage = cv.CreateMemStorage(0)
seq = cv.SegmentMotion(mhi, segmask, storage, timestamp, MAX_TIME_DELTA)
for (area, value, comp_rect) in seq:
if comp_rect[2] + comp_rect[3] > 100: # reject very small components
color = cv.CV_RGB(255, 0,0)
silh_roi = cv.GetSubRect(silh, comp_rect)
mhi_roi = cv.GetSubRect(mhi, comp_rect)
orient_roi = cv.GetSubRect(orient, comp_rect)
mask_roi = cv.GetSubRect(mask, comp_rect)
angle = 360 - cv.CalcGlobalOrientation(orient_roi, mask_roi, mhi_roi, timestamp, MHI_DURATION)
count = cv.Norm(silh_roi, None, cv.CV_L1, None) # calculate number of points within silhouette ROI
if count < (comp_rect[2] * comp_rect[3] * 0.05):
continue
magnitude = 30.
center = ((comp_rect[0] + comp_rect[2] / 2), (comp_rect[1] + comp_rect[3] / 2))
cv.Circle(dst, center, cv.Round(magnitude*1.2), color, 3, cv.CV_AA, 0)
cv.Line(dst,
center,
(cv.Round(center[0] + magnitude * cos(angle * cv.CV_PI / 180)),
cv.Round(center[1] - magnitude * sin(angle * cv.CV_PI / 180))),
color,
3,
cv.CV_AA,
0)
def process_motion(self,img):
center = (-1, -1)
# a lot of stuff from this section was taken from the code motempl.py,
# openCV's python sample code
timestamp = time.clock() / self.clocks_per_sec # get current time in seconds
idx1 = self.last
cv.CvtColor(img, self.buf[self.last], cv.CV_BGR2GRAY) # convert frame to grayscale
idx2 = (self.last + 1) % self.n_frames
self.last = idx2
silh = self.buf[idx2]
cv.AbsDiff(self.buf[idx1], self.buf[idx2], silh) # get difference between frames
cv.Threshold(silh, silh, 30, 1, cv.CV_THRESH_BINARY) # and threshold it
cv.UpdateMotionHistory(silh, self.mhi, timestamp, self.mhi_duration) # update MHI
cv.ConvertScale(self.mhi, self.mask, 255./self.mhi_duration,
(self.mhi_duration - timestamp)*255./self.mhi_duration)
cv.SetZero(img)
cv.Merge(self.mask, None, None, None, img)
cv.CalcMotionGradient(self.mhi, self.mask, self.orient, self.max_time_delta, self.min_time_delta, 3)
seq = cv.SegmentMotion(self.mhi, self.segmask, self.storage, timestamp, self.max_time_delta)
inc = 0
a_max = 0
max_rect = -1
# there are lots of things moving around
# in this case just find find the biggest change on the image
for (area, value, comp_rect) in seq:
if comp_rect[2] + comp_rect[3] > 60: # reject small changes
if area > a_max:
a_max = area
max_rect = inc
inc += 1
# found it, now just do some processing on the area.
if max_rect != -1:
(area, value, comp_rect) = seq[max_rect]
color = cv.CV_RGB(255, 0,0)
silh_roi = cv.GetSubRect(silh, comp_rect)
# calculate number of points within silhouette ROI
count = cv.Norm(silh_roi, None, cv.CV_L1, None)
# this rectangle contains the overall motion ROI
cv.Rectangle(self.motion, (comp_rect[0], comp_rect[1]),
(comp_rect[0] + comp_rect[2],
comp_rect[1] + comp_rect[3]), (0,0,255), 1)
# the goal is to report back a center of movement contained in a rectangle
# adjust the height based on the number generated by the slider bar
h = int(comp_rect[1] + (comp_rect[3] * (float(self.height_value) / 100)))
# then calculate the center
center = ((comp_rect[0] + comp_rect[2] / 2), h)
return center
def plot_selected_hist(hist, image_name='', L=256, hist_type='polylines'):
height = 300
l1_norm_min = cv.Norm(cv.fromarray(hist), None, cv2.NORM_L1)
params = (height, 0, cv2.NORM_INF)
cv2.normalize(hist, hist, *params)
hist = np.int32(np.around(hist))
# density of probability calculation
bins = np.arange(L) #.reshape(256,1)
pts = np.column_stack((bins, height - hist))
hist_image = np.zeros((height, L, 1))
if hist_type == 'polylines':
cv2.polylines(hist_image, [pts], False, WHITE)
else:
for (x, y) in pts:
cv2.line(hist_image, (x, y), (x, height), WHITE)
cv2.imshow(image_name, hist_image)
return image_name
def plot_hist(image, mask=None, image_name='', hist_type='polylines'):
bins = np.arange(256) #.reshape(256,1)
slices = cv2.split(image)
colors = zip(('b', 'g', 'r'), slices)
color_dict = {'b': (255, 0, 0), 'g': (0, 255, 0), 'r': (0, 0, 255)}
winnames = []
l1_norm = []
subhists = []
height = 300
for i, (color, slice) in enumerate(colors):
#cv2.imshow(color, slice)
subhist = cv2.calcHist([slice], [0], mask, [256], [0, 255])
subhists.append(subhist)
params = (0, height - 1, cv2.NORM_MINMAX)
cv2.normalize(subhist, subhist, *params)
l1_norm.append(cv.Norm(cv.fromarray(subhist), None, cv2.NORM_L1))
l1_norm_min = min(l1_norm)
hist_image = np.zeros((height, 256, 3))
for i, (color, slice) in enumerate(colors):
subhist = subhists[i]
params = (l1_norm_min, 0, cv2.NORM_L1)
cv2.normalize(subhist, subhist, *params)
subhist = np.int32(np.around(subhist))
# density of probability calculation
subhists[i] = map(lambda x: float(x) / l1_norm_min, subhist)
pts = np.column_stack((bins, height - subhist))
if hist_type == 'polylines':
cv2.polylines(hist_image, [pts], False, color_dict[color])
else:
for (x, y) in pts:
cv2.line(hist_image, (x, y), (x, height), color_dict[color])
winname = '%s %s' % (image_name, color)
winnames.append(winname)
cv2.imshow(winname, hist_image)
#color_triangle = plot_color_triangle(image, mask)
#cv.ShowImage('%s color triangle: '%image_name, color_triangle)
#color_rectangle = plot_color_rectangle(image, mask)
#cv.ShowImage('%s color triangle: '%image_name, color_rectangle)
return subhists, winnames
def segment_rect(image,
rect,
debug=False,
display=None,
target_size=None,
group_range=(3, 25)):
global next
skip = False
best_chars = []
best_threshold = None
thresholded = cv.CloneImage(image)
contour_image = cv.CloneImage(image)
edges = cv.CloneImage(image)
min_x, min_y, width, height = rect
# cv.SetImageROI(thresholded, rect)
cv.SetImageROI(contour_image, rect)
cv.SetImageROI(image, rect)
cv.SetImageROI(edges, rect)
horizontal = cv.CreateImage(cv.GetSize(image), cv.IPL_DEPTH_16S, 1)
magnitude32f = cv.CreateImage(cv.GetSize(image), cv.IPL_DEPTH_32F, 1)
vertical = cv.CloneImage(horizontal)
magnitude = cv.CloneImage(horizontal)
cv.Sobel(image, horizontal, 0, 1, 3)
cv.Sobel(image, vertical, 1, 0, 3)
cv.Pow(horizontal, horizontal, 2)
cv.Pow(vertical, vertical, 2)
cv.Add(vertical, horizontal, magnitude)
cv.Convert(magnitude, magnitude32f)
cv.Pow(magnitude32f, magnitude32f, 0.5)
cv.Convert(magnitude32f, edges)
original_rect = rect
if display:
cv.SetImageROI(display, rect)
for threshold in range(1, 20, 1):
cv.SetImageROI(thresholded, original_rect)
#for i in range(30, 60, 1):
if display:
cv.Merge(image, image, image, None, display)
cv.Copy(image, thresholded)
#cv.Threshold(thresholded, thresholded, i, 255, cv.CV_THRESH_BINARY_INV)
cv.AdaptiveThreshold(thresholded, thresholded, 255,
cv.CV_ADAPTIVE_THRESH_MEAN_C,
cv.CV_THRESH_BINARY_INV, 17, threshold)
#cv.AdaptiveThreshold(thresholded, thresholded, 255, cv.ADAPTIVE_THRESH_GAUSSIAN_C, cv.THRESH_BINARY_INV, 5, i)
# skip rects greater than 50% thresholded
summed = cv.Norm(thresholded, None, cv.CV_L1,
None) / 255 / thresholded.width / thresholded.height
if summed > 0.5:
continue
if debug:
cv.ShowImage("edge", thresholded)
storage = cv.CreateMemStorage(0)
cv.Copy(thresholded, contour_image)
contours = cv.FindContours(contour_image, storage, cv.CV_RETR_LIST,
cv.CV_CHAIN_APPROX_SIMPLE, (0, 0))
ext.filter_contours(contours, 20, ext.LESSTHAN)
groups = []
rects = []
edge_counts = []
overlappings = {}
if contours:
seq = contours
while seq:
c = ext.as_contour(ext.wrapped(seq))
r = (c.rect.x, c.rect.y, c.rect.width, c.rect.height)
rects.append(r)
seq = seq.h_next()
similarity = 0.45 #0.3
rects.sort(lambda x, y: cmp(y[2] * y[3], x[2] * x[3]))
for rect in rects:
if debug:
print
print "R", rect, len(groups)
cv.SetImageROI(edges,
(original_rect[0] + rect[0],
original_rect[1] + rect[1], rect[2], rect[3]))
edge_count = cv.Sum(edges)[0] / 255 / (rect[2] * rect[3])
edge_counts.append(edge_count)
# cv.ShowImage("edges", edges)
# cv.WaitKey(0)
if debug and target_size:
print "X", target_size, rect
print(target_size[0] - rect[2]) / target_size[0]
print(target_size[1] - rect[3]) / target_size[1]
if rect[2] > rect[3] or float(rect[3])/rect[2] < 3./3 or edge_count < 0.1\
or (rect[2] == image.width and rect[3] == image.height) \
or (target_size and not 0 < (target_size[0] - rect[2]) / target_size[0] < 0.3 \
and not 0 < (target_size[1] - rect[3]) / target_size[1] < 0.05):
if debug:
print "rej", rect[2], ">", rect[3], "edge=", edge_count
cv.Rectangle(display, (rect[0], rect[1]),
(rect[0] + rect[2], rect[1] + rect[3]),
(0, 0, 255), 1)
cv.ShowImage("main", display)
if not skip and not next:
c = cv.WaitKey(0)
if c == ord("a"):
skip = True
if c == ord("z"):
next = True
continue
added = False
for group_id, group in enumerate(groups):
avg_width, avg_height, avg_y = 0, 0, 0
overlap = None
c = 0
for r in group:
avg_y += r[1] + r[3] / 2.0
avg_width += r[2]
avg_height += r[3]
irect = intersect(r, rect)
if irect[2] * irect[3] > 0.2 * r[2] * r[3]:
overlappings.setdefault(group_id,
[]).append([r, rect])
avg_y /= float(len(group))
avg_width /= float(len(group))
avg_height /= float(len(group))
if debug:
print group
if (abs(avg_width - rect[2]) / avg_width < similarity or \
(rect[2] < avg_width)) and \
abs(avg_height - rect[3])/ avg_height < similarity and \
abs(avg_y - (rect[1] + rect[3]/2.0)) / avg_y < similarity:
group.append(rect)
added = True
else:
pass
if not added:
# first char in group
groups.append([rect])
if debug:
print "now:"
for g in groups:
print g
cv.Rectangle(display, (rect[0], rect[1]),
(rect[0] + rect[2], rect[1] + rect[3]),
(255, 0, 0), 1)
cv.ShowImage("main", display)
if not skip and not next:
c = cv.WaitKey(0)
if c == ord("a"):
skip = True
if c == ord("z"):
next = True
if groups:
#handle overlapping regions, default to average width match
for group_id, over in overlappings.items():
group = groups[group_id]
avg_width = 0
avg_height = 0
for r in group:
avg_width += r[2]
avg_height += r[3]
avg_width /= float(len(group))
avg_height /= float(len(group))
for r1, r2 in over:
if r2 not in group or r1 not in group:
continue
if debug:
print "over", r1, r2, r1[2] * r1[3], r2[2] * r2[
3], avg_width
d1 = abs(r1[2] - avg_width) + abs(r1[3] - avg_height)
d2 = abs(r2[2] - avg_width) + abs(r2[3] - avg_height)
if d1 < d2:
group.remove(r2)
else:
group.remove(r1)
#group = max(groups, key=len)
# from longest groups, find largest area
groups.sort(key=len)
groups.reverse()
max_area = 0
mad_index = -1
for i, g in enumerate(groups[:5]):
area = 0
for r in g:
area += r[2] * r[3]
if area > max_area:
max_area = area
max_index = i
group = groups[max_index]
# vertical splitting
avg_width, avg_height, avg_y = 0, 0, 0
if debug:
print "G", group
for r in group:
avg_y += r[1] + r[3] / 2.0
avg_width += r[2]
avg_height += r[3]
avg_y /= float(len(group))
avg_width /= float(len(group))
avg_height /= float(len(group))
band_rects = []
bound = bounding_rect(group)
for i, rect in enumerate(rects):
if edge_counts[i] < 0.1:
continue
if (abs(avg_width - rect[2]) / avg_width < similarity or \
(rect[2] < avg_width)) and \
(abs(avg_height - rect[3]) / avg_height < similarity or \
(rect[3] < avg_height)) and \
abs(avg_y - (rect[1] + rect[3]/2.0)) < avg_height/2:
band_rects.append(rect)
band_rects.sort(lambda x, y: cmp(y[2] * y[3], x[2] * x[3]))
for i, rect_a in enumerate(band_rects[:-1]):
if rect_a[2] * rect_a[3] < 0.2 * avg_width * avg_height:
continue
merge_rects = []
for rect_b in band_rects[i + 1:]:
w = avg_width
m1 = rect_a[0] + rect_a[2] / 2
m2 = rect_b[0] + rect_b[2] / 2
if abs(m1 - m2) < w:
merge_rects.append(rect_b)
if debug:
print "M", merge_rects
if merge_rects:
merge_rects.append(rect_a)
rect = bounding_rect(merge_rects)
area = 0
for r in merge_rects:
area += r[2] * r[3]
if (abs(avg_width - rect[2]) / avg_width < similarity or \
(rect[2] < avg_width)) and \
abs(avg_height - rect[3])/ avg_height < similarity and \
area > 0.5*(avg_width*avg_height) and \
abs(avg_y - (rect[1] + rect[3]/2.0)) / avg_y < similarity:
for r in merge_rects:
if r in group:
group.remove(r)
# merge into group
new_group = []
merged = False
for gr in group:
area2 = max(gr[2] * gr[3], rect[2] * rect[3])
isect = intersect(gr, rect)
if isect[2] * isect[3] > 0.4 * area2:
x = min(gr[0], rect[0])
y = min(gr[1], rect[1])
x2 = max(gr[0] + gr[2], rect[0] + rect[2])
y2 = max(gr[1] + gr[3], rect[1] + rect[3])
new_rect = (x, y, x2 - x, y2 - y)
new_group.append(new_rect)
merged = True
else:
new_group.append(gr)
if not merged:
new_group.append(rect)
group = new_group
cv.Rectangle(display, (rect[0], rect[1]),
(rect[0] + rect[2], rect[1] + rect[3]),
(255, 0, 255), 2)
# avoid splitting
split = False
# select higher threshold if innovates significantly
best_width = 0.0
if best_chars:
best_area = 0.0
for rect in best_chars:
best_area += rect[2] * rect[3]
best_width += rect[2]
best_width /= len(best_chars)
area = 0.0
overlapped = 0.0
avg_width = 0.0
avg_height = 0.0
for rect in group:
area += rect[2] * rect[3]
avg_width += rect[2]
avg_height += rect[3]
for char in best_chars:
section = intersect(rect, char)
if section[2] * section[3] > 0:
overlapped += section[2] * section[3]
avg_width /= len(group)
avg_height /= len(group)
quotient = overlapped / area
quotient2 = (area - overlapped) / best_area
if debug:
print area, overlapped, best_area
print group
print "QUO", quotient
print "QUO2", quotient2
else:
quotient = 0
quotient2 = 1
best_area = 0
group.sort(lambda x, y: cmp(x[0] + x[2] / 2, y[0] + y[2] / 2))
best_chars.sort(lambda x, y: cmp(x[0] + x[2] / 2, y[0] + y[2] / 2))
if group_range[0] <= len(group) <= group_range[1] and avg_width > 5 and avg_height > 10 and \
((quotient2 > 0.05 and (best_area == 0 or abs(area - best_area)/best_area < 0.4))
or (quotient2 > 0.3 and area > best_area)):
if debug:
print "ASSIGNED", group
best_chars = group
best_threshold = threshold #get_patch(thresholded, original_rect)
else:
if debug:
print "not", quotient2, len(
group), avg_width, avg_height, area, best_area
# best_chars = groups
if debug:
for rect in best_chars:
cv.Rectangle(display, (rect[0], rect[1]),
(rect[0] + rect[2], rect[1] + rect[3]),
(0, 255, 0), 1)
cv.ShowImage("main", display)
if not skip and not next:
c = cv.WaitKey(0)
if c == ord("a"):
skip = True
if c == ord("z"):
next = True
best_chars.sort(lambda x, y: cmp(x[0], y[0]))
cv.ResetImageROI(thresholded)
cv.ResetImageROI(contour_image)
cv.ResetImageROI(image)
cv.ResetImageROI(edges)
if display:
cv.ResetImageROI(display)
return best_chars, best_threshold