def Benchmark():
global captureQ, frameCountQ
global captureR, frameCountR
# Doing benchmarking
while True:
if config.OCV_OLD_PY_BINDINGS:
frame1 = captureQ.get(cv2.cv.CV_CAP_PROP_POS_FRAMES)
else:
frame1 = captureQ.get(cv2.CAP_PROP_POS_FRAMES)
common.DebugPrint("Alex: frame1 = %d" % frame1)
MatchFrames.counterQ = int(frame1) #0
common.DebugPrint("Alex: MatchFrames.counterQ = %d" %
MatchFrames.counterQ)
retQ, imgQ = captureQ.read()
if retQ == False:
break
if False:
grayQ = common.ConvertImgToGrayscale(imgQ)
MatchFrames.Main_img1(imgQ, MatchFrames.counterQ)
# 36.2 secs (38.5 secs with Convert to RGB)
common.DebugPrint("Alex: time after Feature Extraction of all frames of " \
"video 1 = %s" % GetCurrentDateTimeStringWithMilliseconds())
while True:
if config.OCV_OLD_PY_BINDINGS:
frameR = captureR.get(cv2.cv.CV_CAP_PROP_POS_FRAMES)
else:
frameR = captureR.get(cv2.CAP_PROP_POS_FRAMES)
common.DebugPrint("Alex: frameR = %d" % frameR)
MatchFrames.counterR = int(frameR)
#0
common.DebugPrint("Alex: counterR = %d" % (MatchFrames.counterR))
retR, imgR = captureR.read()
if retR == False:
break
if False:
grayR = common.ConvertImgToGrayscale(imgR)
MatchFrames.Main_img2(imgR, MatchFrames.counterR)
# Note: 47.2 secs (56.7 secs with Convert to RGB)
common.DebugPrint("Alex: time after Feature Extraction of all frames of " \
"video 2 and (FLANN?) matching once for each frame = %s" % \
GetCurrentDateTimeStringWithMilliseconds())
quit()
def MyImageReadMSH(index):
global g
"""
common.DebugPrint("MyImageReadMSH(): initFrame = %d)" % \
(config.initFrame[g.indexVideo]));
index += config.initFrame[g.indexVideo];
"""
"""
common.DebugPrint("Entered MyImageReadMSH(capture=%s, index=%s)" % \
(str(capture), str(index)));
"""
common.DebugPrint("Entered MyImageReadMSH(index=%s)" % \
(str(index)))
"""
We must reopen the capture device in each different process, otherwise the
program blocks at the first operation in the "global" capture device.
"""
if g.harlocsFolder.endswith(config.HARRIS_QUERY_FOLDER_NAME):
if g.captureQ == None:
capture = cv2.VideoCapture(sys.argv[1])
g.captureQ = capture
common.DebugPrint("MyImageReadMSH(): new capture=%s" % \
(str(capture)))
else:
capture = g.captureQ
elif g.harlocsFolder.endswith(config.HARRIS_REFERENCE_FOLDER_NAME):
if g.captureR == None:
capture = cv2.VideoCapture(sys.argv[2])
g.captureR = capture
common.DebugPrint("MyImageReadMSH(): new capture=%s" % \
(str(capture)))
else:
capture = g.captureR
else:
assert False
assert (g.indexVideo == 0) or (g.indexVideo == 1)
if config.OCV_OLD_PY_BINDINGS:
capture.set(cv2.cv.CV_CAP_PROP_POS_FRAMES, \
index)
else:
"""
From http://docs.opencv.org/modules/highgui/doc/reading_and_writing_images_and_video.html#videocapture-get:
<<CV_CAP_PROP_POS_FRAMES 0-based index of the frame to be
decoded/captured next.>>
"""
capture.set(cv2.CAP_PROP_POS_FRAMES, \
index)
common.DebugPrint("MyImageReadMSH(): after capture.set()")
# This is only for (paranoid) testing purposes:
if config.OCV_OLD_PY_BINDINGS:
indexCrt = capture.get(cv2.cv.CV_CAP_PROP_POS_FRAMES)
else:
"""
From http://docs.opencv.org/modules/highgui/doc/reading_and_writing_images_and_video.html#videocapture-get:
<<CV_CAP_PROP_POS_FRAMES 0-based index of the frame to be
decoded/captured next.>>
"""
indexCrt = capture.get(cv2.CAP_PROP_POS_FRAMES)
#assert int(indexCrt) == index;
#!!!!TODO: think if OK
if int(indexCrt) != index:
common.DebugPrint( \
"MyImageReadMSH(): indexCrt != index --> returning black frame")
ret = False
else:
#common.DebugPrint("Alex: frameR = %d" % frameR);
#if myIndex > numFramesR:
# break;
#ret, img = r_path.read();
ret, img = capture.read()
#if ret == False:
# break;
#!!!!TODO: think if well
#assert ret == True;
if ret == False:
common.DebugPrint(
"MyImageReadMSH(index=%d): ret == False --> returning None" %
index)
img = None
else:
common.DebugPrint("MyImageReadMSH(): img.shape = %s" % str(img.shape))
common.DebugPrint("MyImageReadMSH(): img.dtype = %s" % str(img.dtype))
#!!!!TODO: I suggest to do the gray conversion at reading, not in multi_scale_harris.py
if False:
# In the Matlab code he reads gray/8bpp JPEGs
imgGray = common.ConvertImgToGrayscale(img)
if config.VIDEO_FRAME_RESIZE_SCALING_FACTOR != 1:
# We resize the image
img = Matlab.imresize(img, \
scale=config.VIDEO_FRAME_RESIZE_SCALING_FACTOR)
common.DebugPrint("Exiting MyImageReadMSH()")
if False:
return imgGray
return img
def output_crossref_images(crossref, capture_q, capture_r):
if not os.path.exists(config.FRAME_PAIRS_MATCHES_FOLDER):
os.makedirs(config.FRAME_PAIRS_MATCHES_FOLDER)
rframe = None
gray_rframe = None
last_r_idx = None
t0 = float(cv2.getTickCount())
for q_idx, r_idx in crossref:
q_idx = int(q_idx)
r_idx = int(r_idx)
if common.MY_DEBUG_STDOUT:
t1 = float(cv2.getTickCount())
qframe = MyImageRead(capture_q, q_idx, grayscale=False)
gray_qframe = common.ConvertImgToGrayscale(qframe)
if r_idx != last_r_idx:
rframe = MyImageRead(capture_r, r_idx, grayscale=False)
gray_rframe = common.ConvertImgToGrayscale(rframe)
last_r_idx = r_idx
if common.MY_DEBUG_STDOUT:
t2 = float(cv2.getTickCount())
common.DebugPrint(
"ecc_homo_spacetime(): grabbing frames took %.6f [sec]" %
((t2 - t1) / cv2.getTickFrequency()))
# Default diff is a G-channel swap.
gdiff = np.zeros(qframe.shape)
gdiff[:, :, 0] = gray_qframe
gdiff[:, :, 1] = gray_rframe # G-Channel Swap
gdiff[:, :, 2] = gray_qframe
if config.VISUAL_DIFF_FRAMES:
# Compute difference between frames.
diff_mask = cv2.absdiff(qframe, rframe)
for i in range(diff_mask.shape[2]):
diff_mask[:, :, i] = cv2.GaussianBlur(src=diff_mask[:, :, i],
ksize=(7, 7),
sigmaX=10)
diff_mask_sum = diff_mask.sum(axis=2)
r_diff, c_diff = np.nonzero(
diff_mask_sum >= config.MEANINGFUL_DIFF_THRESHOLD *
diff_mask.shape[2])
meaningful_indices = zip(r_diff, c_diff)
# Create all black image so we can set the diff pixels to white.
diff_mask = np.zeros((diff_mask.shape[0], diff_mask.shape[1]),
dtype=np.uint8)
diff_mask[(r_diff, c_diff)] = 255
assert r_diff.size == c_diff.size
assert r_diff.size == len(meaningful_indices)
res = cv2.findContours(image=diff_mask,
mode=cv2.RETR_TREE,
method=cv2.CHAIN_APPROX_SIMPLE)
contours = res[1]
color_c = 255
meaningful_contours = 0
diff_mask = np.zeros((diff_mask.shape[0], diff_mask.shape[1]),
dtype=np.uint8)
for index_c, contour in enumerate(contours):
if len(contour) < 15:
continue
else:
cv2.drawContours(image=diff_mask,
contours=[contour],
contourIdx=0,
color=color_c,
thickness=-1)
meaningful_contours += 1
cv2.imwrite(
config.FRAME_PAIRS_MATCHES_FOLDER +
("%.6d_diff_mask" % q_idx) + config.imformat,
diff_mask.astype(int))
if config.SHOW_MASKED_DIFF:
qframe_diff = cv2.bitwise_and(qframe, qframe, mask=diff_mask)
rframe_diff = cv2.bitwise_and(rframe, rframe, mask=diff_mask)
# Save the masked diffs.
cv2.imwrite(
config.FRAME_PAIRS_MATCHES_FOLDER +
("%.6d_query_diff" % q_idx) + config.imformat,
qframe_diff.astype(int))
cv2.imwrite(
config.FRAME_PAIRS_MATCHES_FOLDER +
("%.6d_ref_diff" % q_idx) + config.imformat,
rframe_diff.astype(int))
if config.SAVE_FRAMES:
# We use q_idx because ref video is aligned against the query video.
# These frames represent the aligned output.
cv2.imwrite(
config.FRAME_PAIRS_MATCHES_FOLDER + ("%.6d_query" % q_idx) +
config.imformat, qframe.astype(int))
cv2.imwrite(
config.FRAME_PAIRS_MATCHES_FOLDER + ("%.6d_ref" % q_idx) +
config.imformat, rframe.astype(int))
cv2.imwrite(
config.FRAME_PAIRS_MATCHES_FOLDER + ("%.6d_gchan_diff" % q_idx) +
config.imformat, gdiff.astype(int))
print("Saving crossref frames took %.6f [sec]" %
((float(cv2.getTickCount()) - t0) / cv2.getTickFrequency()))
def spatial_alignment(win,
input_frame,
ref_frame,
kp_pairs,
status=None,
h=None):
h_q, w_q = input_frame.shape[:2]
h_r, w_r = ref_frame.shape[:2]
vis = None
# Alex: if it's an inlier we draw a line, etc
"""
Alex: we convert images to gray, otherwise we get exception like:
Traceback (most recent call last):
File "ReadAVI.py", line 321, in <module>
Main()
File "ReadAVI.py", line 258, in Main
res = find_obj.ComputeFeatures2(ref_frame, counter_r)
File "Z:\1PhD\UPB_RO\CV_Video_Based_Detection\1\find_obj.py", line 387, in ComputeFeatures2
res = temporal_alignment('find_obj')
File "Z:\1PhD\UPB_RO\CV_Video_Based_Detection\1\find_obj.py", line 334, in temporal_alignment
vis = spatial_alignment(win, input_frame, ref_frame, kp_pairs, status, H)
File "Z:\1PhD\UPB_RO\CV_Video_Based_Detection\1\find_obj.py", line 161, in spatial_alignment
vis[:hQ, :wQ] = input_frame
ValueError: operands could not be broadcast together with shapes (576,720) (576,720,3)
"""
"""
Note that the formal parameters of the caller corresponding to the actual
parameters input_frame and ref_frame are not changed by the following
assignments:
"""
input_frame = common.ConvertImgToGrayscale(input_frame)
ref_frame = common.ConvertImgToGrayscale(ref_frame)
display_warped_image = False
if config.SPATIAL_ALIGNMENT_ALGO in [
"LK", "TEMPORAL_ALIGNMENT_HOMOGRAPHY"
]:
if config.SPATIAL_ALIGNMENT_ALGO == "LK":
# Computing homography with Lucas-Kanade's algorithm:
h1, status1 = LK.lucas_kanade_homography(input_frame)
if h1 is not None:
# We take the H from the LK algorithm
h = h1
status = status1
if config.USE_GUI or config.SAVE_FRAMES:
# Applying overlay of warped ref_frame with homography H
h, w = ref_frame.shape[:2]
overlay = cv2.warpPerspective(src=ref_frame, M=h, dsize=(w, h))
vis = np.zeros((max(h_q, h_r), w_q + w_r), np.uint8)
vis[:h_q, :w_q] = input_frame
if config.DISPLAY_RIGHT_IMAGE_WITH_DIFF:
# ref_frame is the warped ref_frame, when applying H to it
ref_frame = overlay
# TODO: using img_diff = None seems to crash at absdiff().
# This is just a crappy assignment, to be of the same type :))
img_diff = ref_frame
# Compute difference between frames input_frame and ref_frame:
cv2.absdiff(src1=input_frame, src2=ref_frame, dst=img_diff)
vis[:h_r, w_q:w_q + w_r] = img_diff
else:
if display_warped_image:
ref_frame = overlay
vis[:h_r, w_q:w_q + w_r] = ref_frame
elif config.SPATIAL_ALIGNMENT_ALGO == "ECC":
ECC.template_image = input_frame
ECC.target_image = ref_frame
ECC.number_of_iterations = 25
ECC.termination_eps = 0.001
ECC.motion = "homography"
ECC.warp_to_file = "warp.ecc"
ECC.warp_init = "init.txt"
ECC.image_to_file = "warped.pgm"
# Note that ECC.template_image is a numpy.ndarray
ECC.target_image = Misc.convert_np_to_ipl_image(ECC.target_image)
ECC.template_image = Misc.convert_np_to_ipl_image(ECC.template_image)
if config.USE_GUI or config.SAVE_FRAMES:
vis = np.zeros((max(h_q, h_r), w_q + w_r), np.uint8)
vis[:h_q, :w_q] = input_frame
vis[:h_r, w_q:w_q + w_r] = ref_frame
if config.USE_GUI:
cv2.imshow(win, vis)
cv2.waitKey(2000)
# We require IplImage; the result is basically ECC.warped_image
ECC.main()
if config.USE_GUI or config.SAVE_FRAMES:
# TODO: using img_diff = None seems to crash at absdiff().
# This is just a crappy assignment, to be of the same type :))
img_diff = ref_frame
ref_frame_new = ECC.warped_image
print("ref_frame_new (before) = %s" % str(ref_frame_new))
"""
# We now need to convert ref_frame_new to numpy:
From https://stackoverflow.com/questions/13104161/fast-conversion-of-iplimage-to-numpy-array:
"works fantastically, and more important: quickly!
This is by far the fastest way I've found to grab a frame and make
it an ndarray for numpy processing." - see code below:
"""
ref_frame_new = np.asarray(ref_frame_new[:, :])
print("input_frame = %s" % str(input_frame))
print("ref_frame = %s" % str(ref_frame))
print("ref_frame_new = %s" % str(ref_frame_new))
"""
Because both input_frame and ref_frame are
OLD:
input_frame =
<iplimage(nChannels=1 width=568 height=320 widthStep=568 )>
img_diff =
<iplimage(nChannels=1 width=568 height=320 widthStep=568 )>
**
input_frame =
<iplimage(nChannels=1 width=568 height=320 widthStep=568 )>
ref_frame = [[47 46 47 ..., 59 49 39]
[47 47 46 ..., 61 49 39]
[46 47 46 ..., 61 49 38]
...,
[85 85 86 ..., 79 79 79]
[86 86 88 ..., 82 82 82]
[88 88 88 ..., 84 83 84]]
**
input_frame =
<iplimage(nChannels=1 width=568 height=320 widthStep=568 )>
ref_frame =
<iplimage(nChannels=1 width=568 height=320 widthStep=568 )>
We have the following error:
"TypeError: src1 is not a numpy array, neither a scalar"
"""
if config.DISPLAY_RIGHT_IMAGE_WITH_DIFF:
# Compute difference between frames input_frame and ref_frame:
cv2.absdiff(src1=input_frame, src2=ref_frame_new, dst=img_diff)
else:
img_diff = ref_frame_new
"""
We display in the right pane the diff between the warped image of
the target image and the template?
"""
vis[:h_r, w_q:w_q + w_r] = img_diff
"""
TODO: return reduced_feature_set - feature_set keypoints of input frame
that overlap with ref_frame.
# TODO: in many cases the corners after applying H are just 1 point or a
line or a very narrow quadrilater - also the feature points
# We now draw the feature points of the INPUT frame on which we apply H
ft = np.float32([kpp[1].pt for kpp in kp_pairs]) #!!!!TODO: should be
kpp[0].pt, NOT the features of the reference frame
print "ft = %s" % str(ft)
ftT = cv2.perspectiveTransform(ft.reshape(1, -1, 2), H)
print "ftT = %s" % str(ftT)
ft = np.int32(ftT.reshape(-1, 2) + (wQ, 0))
print "ft (after) = %s" % str(ft)
for e in ft:
if keypoint (e[0], e[1]) inside the homography of the rectangle of the
frame then the keypoint is in reduced_feature_set
"""
return vis, input_frame, ref_frame, status
def multi_scale_harris(im, nos, disp):
"""
Followed advices from emails:
- Vali Codreanu, Mar 18th, 2014:
- Evangelidis, Apr th, 2014:
"""
common.DebugPrint("Entered multi_scale_harris(nos=%d, disp=%s)" % \
(nos, str(disp)))
t1 = float(cv2.getTickCount())
# nos = number of scales
im = common.ConvertImgToGrayscale(im)
#print "feature_params (a dictionary) = %s" % str(feature_params)
feature_params = dict(
maxCorners=1000,
qualityLevel=0.01,
minDistance=
5, #9, #7, #~195 points, #6, #~210 points #3, #~300 points, #4, #~100 points, #5,#~85 Harris points found #8, ~45 found
blockSize=19,
useHarrisDetector=True)
"""
Alex: added useHarrisDetector for multi-scale harris detector inspired
from Evangelidis, (otherwise it was using cornerMinEigenVal detector).
"""
if False:
pointsAux = cv2.goodFeaturesToTrack(im, **feature_params)
points = []
if pointsAux == None:
return points
for p in pointsAux:
points.append((p[0][0], p[0][1], 1))
# we only have scale 1 now
points = []
"""
multi-scale Harris detectors
- http://opencv-users.1802565.n2.nabble.com/multi-scale-corners-detection-td4702476.html
"Not in OpenCV, but you can always code it yourself using the available OpenCV functions.
One approach is to generate a pyramid and apply cvGoodFeatureTracks to each level.
You'll have to remove duplicate detected features though."
- available in http://www.vlfeat.org/
See for example of build image pyramids:
http://opencv-python-tutroals.readthedocs.org/en/latest/py_tutorials/py_imgproc/py_pyramids/py_pyramids.html
Note: cv2.cornerHarris(src, blockSize, ksize, k[, dst[, borderType ]])
"""
# We compute for pyramid of images the Harris features
imScaled = im.copy()
# Note: this is the only place where they have first the height (related to y) and width (related to x) - dimensions are y, x, like matrices
im_h, im_w = im.shape[:2]
"""
Using Evangelidis' scale params (local scale and integration scale) for
multi-scale Harris
"""
# scale values
sigma_0 = 1.2
#n=[0:nos-1]; %scale levels
n = np.array(range(nos))
#scale levels
#sigma_D=sqrt(1.8).^n*sigma_0
sigma_D = math.sqrt(1.8)**n * sigma_0
#scn=sigma_D.^4;
scn = sigma_D**4
common.DebugPrint("multi_scale_harris(): scn = %s" % (str(scn)))
"""
Obtained from Evangelidis' code, we use these ranges to define ksizeGB below:
mRange is basically the 1-D Gaussian kernel
(Evangelidis uses meshgrid(mRange, mRange) to build a 2-D Gaussian kernel).
mRange = [-5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5]
mRange = [-5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5]
mRange = [-7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7]
mRange = [-9, -8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
mRange = [-13, -12, -11, -10, -9, -8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
"""
ksizeGB = [11, 11, 15, 19, 27]
# kernel size GaussianBlur
for i in range(nos):
common.DebugPrint("multi_scale_harris(): i = %s" % (str(i)))
#sd=sigma_D(i); %differentiation (local) scale
sd = sigma_D[i]
#%differentiation (local) scale
imScaled_h, imScaled_w = imScaled.shape[:2]
#if False:
if True:
"""
From http://docs.opencv.org/modules/imgproc/doc/filtering.html
Python: cv2.GaussianBlur(src, ksize, sigmaX[, dst[, sigmaY[, borderType]]]) -> dst
where:
ksize - Gaussian kernel size. ksize.width and ksize.height can
differ but they both must be positive and odd.
Or, they can be zero's and then they are computed
from sigma* .
sigmaX - Gaussian kernel standard deviation in X direction.
sigmaY - Gaussian kernel standard deviation in Y direction;
if sigmaY is zero, it is set to be equal to sigmaX,
if both sigmas are zeros, they are computed from
ksize.width and ksize.height , respectively
(see getGaussianKernel() for details); to fully
control the result regardless of possible future
modifications of all this semantics, it is
recommended to specify all of ksize, sigmaX,
and sigmaY.
borderType - pixel extrapolation method
(see borderInterpolate() for details).
NOTE:
Since the 2D Gaussian kernel can be separable on both dimensions,
GaussianBlur() calls getGaussianKernel() for dim X and Y.
Python: cv2.getGaussianKernel(ksize, sigma[, ktype]) -> retval
Parameters:
ksize - Aperture size. It should be odd
( ksize \mod 2 = 1 ) and positive.
sigma - Gaussian standard deviation.
If it is non-positive, it is computed from ksize
as sigma = 0.3*((ksize-1)*0.5 - 1) + 0.8 .
ktype - Type of filter coefficients.
It can be CV_32f or CV_64F.
The function computes and returns the ksize x 1 matrix of
Gaussian filter coefficients:
G_i= \alpha * e^{-(i-(ksize-1)/2)^2/(2*sigma)^2},
where i=0..ksize-1 and \alpha is the scale factor chosen
so that \sum_i G_i=1.
Two of such generated kernels can be passed to sepFilter2D()
Alex: as we can see above, the mean of the Gaussian coefs
are computed with sigma=sigma and mean=0.
REFERENCE explaining the math behind GaussianBlur():
http://opencv-python-tutroals.readthedocs.org/en/latest/py_tutorials/py_imgproc/py_filtering/py_filtering.html
"""
ks = ksizeGB[i]
imScaled = cv2.GaussianBlur(src=imScaled, \
ksize=(ks, ks), sigmaX=sd*sd)
#0.0);
common.DebugPrint("multi_scale_harris(): ks = %s" % (str(ks)))
common.DebugPrint("multi_scale_harris(): sd^2 = %s" %
(str(sd * sd)))
#feature_params["minDistance"] = 5 - (i / 2); // i is the scale in use
# The corners with the minimal eigenvalue less than \texttt{qualityLevel} \cdot \max_{x,y} qualityMeasureMap(x,y) are rejected.
# This is the threshold used in non-maximal supression.
feature_params["qualityLevel"] = 0.001 * scn[i]
# * 30;
feature_params["minDistance"] = 0.0
#100; (Returns very few points)
"""
blockSize param given to cv::goodFeaturesToTrack() is used by:
- cornerHarris, for Sobel filter (as aperture_size parameter);
- boxFilter, which is applied exactly before computing the
Harris measure.
NOTE: ksize of cv::cornerHaris() is set to 3 by
cv::goodFeaturesToTrack() .
From http://docs.opencv.org/modules/imgproc/doc/feature_detection.html?highlight=cornerharris#cornerharris
"ksize - Aperture parameter for the Sobel() operator."
"""
feature_params["blockSize"] = ks
#1!!!!
feature_params["k"] = 0.06
common.DebugPrint("multi_scale_harris(): feature_params = %s" % \
(str(feature_params)))
"""
cv::goodFeaturesToTrack() with Harris option does the following
(http://docs.opencv.org/modules/imgproc/doc/feature_detection.html#goodfeaturestotrack ,
code at https://github.com/Itseez/opencv/blob/451be9ed538bbad6be61a783c2f4fa247fc83930/modules/imgproc/src/featureselect.cpp):
<<The function finds the most prominent corners in the image or
in the specified image region, as described in [Shi94]:
Function calculates the corner quality measure at every source
image pixel using the cornerMinEigenVal() or cornerHarris() .
IMPORTANT: Function performs a non-maximum suppression (the
local maximums in 3 x 3 neighborhood are retained).
The corners with the minimal eigenvalue less than
qualityLevel \cdot \max_{x,y} qualityMeasureMap(x,y) are rejected.
The remaining corners are sorted by the quality measure in the
descending order.
Function throws away each corner for which there is a stronger
corner at a distance less than maxDistance. >>
"""
"""
Python: cv2.goodFeaturesToTrack(image, maxCorners, qualityLevel, minDistance[, corners[, mask[, blockSize[, useHarrisDetector[, k ]]]]]) -> corners
useHarrisDetector - Parameter indicating whether to use a Harris detector (see cornerHarris()) or cornerMinEigenVal()
k - Free parameter of the Harris detector (see page 324 for details on the dst(x, y) formula using k)
From opencv2refman.pdf, page 327, Section 3.7,
"The function finds the most prominent corners in the image or in
the specified image region, as described in [Shi94]"
"""
#pointsAux = cv2.goodFeaturesToTrack(imScaled, **feature_params);
pointsAux = cv2.goodFeaturesToTrack(
imScaled,
maxCorners=3600, #1000,
qualityLevel=0.01,
#qualityLevel=0.001 * scn[i], # * 30;
minDistance=
5, #9, #7, #~195 points, #6, #~210 points #3, #~300 points, #4, #~100 points, #5,#~85 Harris points found #8, ~45 found
blockSize=ks, #19,
useHarrisDetector=True,
k=0.06)
# The corners with the minimal eigenvalue less than \texttt{qualityLevel} \cdot \max_{x,y} qualityMeasureMap(x,y) are rejected.
# This is the threshold used in non-maximal supression.
#feature_params["minDistance"]=0.0); #100; (Returns very few points)
else:
# Inspired from https://stackoverflow.com/questions/18255958/harris-corner-detection-and-localization-in-opencv-with-python
"""
From OpenCV help:
http://docs.opencv.org/modules/imgproc/doc/feature_detection.html#void%20cornerHarris%28InputArray%20src,%20OutputArray%20dst,%20int%20blockSize,%20int%20ksize,%20double%20k,%20int%20borderType%29
Python: cv2.cornerHarris(src, blockSize, ksize, k[, dst[, borderType ] ]) -> dst
"""
cornerImg = cv2.cornerHarris(src=imScaled, blockSize=2, ksize=3, \
k=0.04, borderType=cv2.BORDER_DEFAULT)
if False:
cornerImg = cv2.cornerHarris(src=imScaled, blockSize=19, ksize=3, \
k=0.01, borderType=cv2.BORDER_DEFAULT)
common.DebugPrint("multi_scale_harris(): cornerImg=%s" % \
(str(cornerImg)))
"""
See http://docs.opencv.org/modules/core/doc/operations_on_arrays.html
Python: cv2.normalize(src[, dst[, alpha[, beta[, norm_type[, dtype[, mask]]]]]]) -> dst
"""
cornerImg = cv2.normalize(src=cornerImg, alpha=0, beta=255, \
norm_type=cv2.NORM_MINMAX, \
dtype=cv2.CV_32FC1)
common.DebugPrint( \
"multi_scale_harris(): cornerImg (after normalize)=%s" % \
(str(cornerImg)))
"""
# From http://docs.opencv.org/modules/core/doc/operations_on_arrays.html#convertscaleabs:
Python: cv2.convertScaleAbs(src[, dst[, alpha[, beta]]]) -> dst
"""
cornerImg = cv2.convertScaleAbs(cornerImg)
common.DebugPrint( \
"multi_scale_harris(): cornerImg (after convertScaleAbs)=%s" % \
(str(cornerImg)))
pointsAux = []
"""
Inspired from
http://docs.opencv.org/doc/tutorials/features2d/trackingmotion/harris_detector/harris_detector.html
"""
thresh = 200
#200.0; #10e-06
# iterate over pixels to get corner positions
w, h = imScaled.shape
common.DebugPrint("multi_scale_harris(): w, h = %s" % \
(str((w, h))))
for y in range(0, h):
for x in range(0, w):
#harris = cv2.cv.Get2D( cv2.cv.fromarray(cornerimg), y, x)
#if cornerimg[x,y] > 64:
# Following
if cornerImg[x, y] > thresh:
#common.DebugPrint("corner at ", x, y)
pointsAux.append((x, y))
"""
cv2.circle( cornershow, # dest
(x,y), # pos
4, # radius
(115,0,25) # color
);
"""
if pointsAux == None:
#return points
continue
common.DebugPrint("multi_scale_harris(): len(pointsAux)=%d" % \
(len(pointsAux)))
common.DebugPrint("multi_scale_harris(): len(pointsAux[0])=%d" % \
(len(pointsAux[0])))
common.DebugPrint("multi_scale_harris(): pointsAux=%s" %
(str(pointsAux)))
# Note: pointsAux contain just the (x, y) coordinates of the points like this: [ [[x1, y1]] [[x2, y2]] ]
#common.DebugPrint("multi_scale_harris(): pointsAux = %s" % str(pointsAux));
for p in pointsAux:
points.append((p[0][0] * float(im_w) / imScaled_w,
p[0][1] * float(im_h) / imScaled_h, i + 1))
#points.append((p[0] * float(im_w)/imScaled_w, p[1] * float(im_h)/imScaled_h, i + 1));
sys.stdout.flush()
# Sort the points after 2nd element first and then the 1st element:
def CmpFunc(x, y):
#return x - y
if x[1] > y[1]:
return 1
elif x[1] < y[1]:
return -1
else: #(x[1] == y[1]):
if (x[0] > y[0]):
return 1
elif (x[0] < y[0]):
return -1
else:
return 0
return 0
points.sort(cmp=CmpFunc)
if False:
# We work on the Gaussian pyramid
imScaled = cv2.pyrDown(imScaled)
common.DebugPrint(
"multi_scale_harris(): imScaled dimensions after cv2.pyrDown are %s"
% str(imScaled.shape[:2]))
#TODO: make course grained harris features be ~roughly same number like in the Matlab code - should I increase the minDistance, etc???
#if False:
if True:
feature_params["minDistance"] = 5 - (i / 2)
common.DebugPrint(
"multi_scale_harris(): feature_params[minDistance] = %d" %
feature_params["minDistance"])
points = np.array(points)
#if False:
if True:
common.DebugPrint("multi_scale_harris(): len(points)=%d, points = %s" %
(len(points), str(points)))
"""
if points is not None:
# We display this information only at the first call of this function,
# BUT not in the next calls.
for x, y in points[:,0]:
cv2.circle(img1, (x, y), 5, green, -1)
common_cv.draw_str(img1, (20, 20), \
"feature count (from goodFeaturesToTrack): %d" % len(p))
"""
t2 = float(cv2.getTickCount())
myTime = (t2 - t1) / cv2.getTickFrequency()
##common.DebugPrint("multiscale_quad_tree() took %.6f [sec]" % myTime);
common.DebugPrint("multi_scale_harris() took %.6f [sec]" % myTime)
# We could even call this function when points is list, not an numpy array
#StoreMultiScaleHarrisFeatures("Videos/harloc%", points);
return points
def multi_scale_harris_Evangelidis(im, nos, disp):
common.DebugPrint("Entered multi_scale_harris_Evangelidis(nos=%d, disp=%s)" % \
(nos, str(disp)))
#tic
#if size(im, 3) == 3:
# im = rgb2gray(im)
tMSH1 = float(cv2.getTickCount())
if im.ndim == 3:
im = common.ConvertImgToGrayscale(im)
#im = im2double(im)
# From https://stackoverflow.com/questions/10873824/how-to-convert-2d-float-numpy-array-to-2d-int-numpy-array:
#DO NOT USE: im = im.astype(np.uint8); # This messes up tremendously the computation of harlocs
#im = im.astype(float);
im = im.astype(np.float32)
#!!!!COR is an UNused variable
#COR = zeros(size(im,1), size(im,2), size(im,3))
#COR = np.zeros((im.shape[0], im.shape[1], im.shape[1]));
# scale values
sigma_0 = 1.2
#n=[0:nos-1]; %scale levels
n = np.array(range(nos))
#scale levels
#sigma_D=sqrt(1.8).^n*sigma_0
sigma_D = math.sqrt(1.8)**n * sigma_0
#points=[];
points = np.array([])
#scn=sigma_D.^4;
scn = sigma_D**4
#for i=1:length(sigma_D)
#for i in range(1, len(sigma_D) + 1):
for i in range(1, nos + 1):
common.DebugPrint("multi_scale_harris_Evangelidis(): i = %s" %
(str(i)))
#sd=sigma_D(i); %differentiation (local) scale
sd = sigma_D[i - 1]
#%differentiation (local) scale
#si=sd/.5; %integration scale
si = sd / 0.5
#integration scale
w = 3 * sd
# size for gaussian kernel to compute derivatives: 3 times local_scale
r_w = int(round(w))
common.DebugPrint("multi_scale_harris_Evangelidis(): r_w = %s" % \
(str(r_w)))
#if mod(round(w),2):
if (r_w % 2) == 1:
#[xco,yco] = meshgrid(-round(w):round(w),-round(w):round(w))
mRange = range(-r_w, r_w + 1)
xco, yco = Matlab.meshgrid(mRange, mRange)
else:
#[xco,yco] = meshgrid(-(round(w)+1):round(w)+1,-(round(w)+1):round(w)+1)
mRange = range(-(r_w + 1), r_w + 2)
xco, yco = Matlab.meshgrid(mRange, mRange)
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): xco = %s" % \
str(xco))
common.DebugPrint("multi_scale_harris_Evangelidis(): yco = %s" % \
str(yco))
# Note: even for HD frames, xco.shape = (11, 11) (yco the same)
#arg = -(xco.*xco + yco.*yco) / (2*sd*sd)
#arg = -(xco * xco + yco * yco) / (2.0 * sd * sd);
arg = -(xco**2 + yco**2) / (2.0 * sd * sd)
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): arg = %s" % \
str(arg))
#%2d gaussian kernel
"""
From http://www.mathworks.com/help/matlab/ref/exp.html:
"exp(X) returns the exponential for each element of array X."
"""
#g=exp(arg)/(2*pi*sd^2); #2d gaussian kernel
g = np.exp(arg) / (2.0 * math.pi * pow(sd, 2))
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): g = %s" % \
str(g))
# normalize to suppress any gain
#if sum(g(:))~=0:
g_sum = g.sum()
#if abs(g.sum()) > 1.0e-6:
if abs(g_sum) > 1.0e-6:
#g = g / sum(g(:));
#g = g / float(g.sum());
g /= float(g_sum)
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): sd = %s" %
str(sd))
common.DebugPrint("multi_scale_harris_Evangelidis(): w = %s" %
str(w))
"""
#%Instead of computing derivatives in the filtered image,
we filter the image with the derivatives of the kernel.
"""
#% kernels for gaussian derivatives
#gx=-xco.*g/(sd*sd);
gx = -xco * g / float(sd * sd)
#gy=-yco.*g/(sd*sd);
gy = -yco * g / float(sd * sd)
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): gx = %s" % \
str(gx))
common.DebugPrint("multi_scale_harris_Evangelidis(): gy = %s" % \
str(gy))
"""
multi_scale_harris_Evangelidis(): arg.shape = (11, 11)
multi_scale_harris_Evangelidis(): g.shape = (11, 11)
multi_scale_harris_Evangelidis(): gx.shape = (11, 11)
multi_scale_harris_Evangelidis(): gy.shape = (11, 11)
multi_scale_harris_Evangelidis(): gi.shape = (15, 15)
"""
common.DebugPrint("multi_scale_harris_Evangelidis(): xco.shape = %s" %
str(xco.shape))
common.DebugPrint("multi_scale_harris_Evangelidis(): yco.shape = %s" %
str(yco.shape))
common.DebugPrint("multi_scale_harris_Evangelidis(): arg.shape = %s" %
str(arg.shape))
common.DebugPrint("multi_scale_harris_Evangelidis(): g.shape = %s" %
str(g.shape))
common.DebugPrint("multi_scale_harris_Evangelidis(): gx.shape = %s" %
str(gx.shape))
common.DebugPrint("multi_scale_harris_Evangelidis(): gy.shape = %s" %
str(gy.shape))
#%compute the derivatives
#Ix = imfilter(im, gx, 'replicate');
Ix = Matlab.imfilter(im, gx, "replicate")
#Iy = imfilter(im, gy, 'replicate');
Iy = Matlab.imfilter(im, gy, "replicate")
#% Alex: Ix and Iy have the same size as im
#if True:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): Ix = %s" % \
str(Ix))
common.DebugPrint("multi_scale_harris_Evangelidis(): Iy = %s" % \
str(Iy))
#% gaussian kernel to compute 2nd moment matrix
#if mod(floor(6*si),2) %size: six times the integration scale
if int(math.floor(
6 * si)) % 2 == 1: #size: six times the integration scale
#gi = fspecial('ga',max(1,fix(6*si)), si)
gi = Matlab.fspecial("ga", max(1, Matlab.fix(6 * si)), si)
else:
#gi = fspecial('ga',max(1,fix(6*si)+1), si)
gi = Matlab.fspecial("ga", max(1,
Matlab.fix(6 * si) + 1), si)
#if True:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): gi = %s" % \
str(gi))
common.DebugPrint("multi_scale_harris_Evangelidis(): gi.shape = %s" % \
str(gi.shape))
#Ix2 = imfilter(Ix.^2, gi, 'replicate');
Ix2 = Matlab.imfilter(Ix**2, gi, "replicate")
#Iy2 = imfilter(Iy.^2, gi, 'replicate');
Iy2 = Matlab.imfilter(Iy**2, gi, "replicate")
#Ixy = imfilter(Ix.*Iy, gi, 'replicate');
Ixy = Matlab.imfilter(Ix * Iy, gi, "replicate")
#% Alex: Ix2, Iy2 and Ixy have the same size as im
#if True:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): Ix2 = %s" % \
str(Ix2))
common.DebugPrint("multi_scale_harris_Evangelidis(): Iy2 = %s" % \
str(Iy2))
common.DebugPrint("multi_scale_harris_Evangelidis(): Ixy = %s" % \
str(Ixy))
common.DebugPrint("multi_scale_harris_Evangelidis(): Ix2.dtype = %s" % \
str(Ix2.dtype))
common.DebugPrint("multi_scale_harris_Evangelidis(): Iy2.dtype = %s" % \
str(Iy2.dtype))
common.DebugPrint("multi_scale_harris_Evangelidis(): Ixy.dtype = %s" % \
str(Ixy.dtype))
#%% Cornerness measure
#% Noble measure.
#%
#% M = (Ix2.*Iy2 - Ixy.^2)./(Ix2 + Iy2 + eps);
#% Harris measure.
#M = (Ix2.*Iy2 - Ixy.^2) - .06*(Ix2 + Iy2).^2;
M = (Ix2 * Iy2 - Ixy**2) - 0.06 * (Ix2 + Iy2)**2
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): M.dtype = %s" % \
str(M.dtype))
common.DebugPrint("multi_scale_harris_Evangelidis(): M = %s" % \
str(M))
#% Alex: scn is a vector - see definition above
#% Alex: M has the same size as im
#M = scn(i)*M;
M = scn[i - 1] * M
#thresh = 0.001*max(M(:));
thresh = 0.001 * M.max()
#% imagesc(M==abs(M));axis on;
#% colorbar
#% pause
"""
%keep points that are the maximum in a neighborhood of
radius=round(3*si/2) and are above thresh
%non-maximum supression and subpixel refinement
"""
#[r,c, rsubp, csubp] = my_nms(M, round(3*si/2), thresh);
r, c, rsubp, csubp = my_nms(M, round(3 * si / 2.0), thresh)
if common.MY_DEBUG_STDOUT:
common.DebugPrint(
"multi_scale_harris_Evangelidis(): r.shape = %s" %
str(r.shape))
common.DebugPrint("multi_scale_harris_Evangelidis(): r = %s" %
str(r))
common.DebugPrint(
"multi_scale_harris_Evangelidis(): c.shape = %s" %
str(c.shape))
common.DebugPrint("multi_scale_harris_Evangelidis(): c = %s" %
str(c))
common.DebugPrint(
"multi_scale_harris_Evangelidis(): rsubp.shape = %s" %
str(rsubp.shape))
common.DebugPrint("multi_scale_harris_Evangelidis(): rsubp = %s" %
str(rsubp))
common.DebugPrint(
"multi_scale_harris_Evangelidis(): csubp.shape = %s" %
str(csubp.shape))
common.DebugPrint("multi_scale_harris_Evangelidis(): csubp = %s" %
str(csubp))
#% Alex: r,c, rsubp, csubp seem to always be the same size - and the
#% size I've seen is 56 * 1????
#pp=[rsubp, csubp, i*ones(size(r,1),1)];
pp = np.c_[rsubp, csubp, i * np.ones((r.shape[0], 1))]
#% Alex: here we add more rows (pp) to points below the existing rows of
#% points
#points=[points; pp];
common.DebugPrint(
"multi_scale_harris_Evangelidis(): points.shape = %s" %
str(points.shape))
common.DebugPrint("multi_scale_harris_Evangelidis(): pp.shape = %s" %
str(pp.shape))
if points.size == 0:
# Avoiding exception: "ValueError: arrays must have same number of dimensions"
points = pp
else:
points = np.r_[points, pp]
#toc
if disp:
assert False
# not implemented the display of Harris features
"""
figure;
imshow(im,[]) #hold on
title('corners detected')
for i = range(1, size(points,1) + 1):
rectangle('Position',[points(i,2)-3*points(i,3),points(i,1)-3*points(i,3),...
2*3*points(i,3),2*3*points(i,3)],'Curvature',[1,1],'EdgeColor','w','LineWidth',2)
plot(points(i,2),points(i,1),'r+')
"""
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): points = %s" %
str(points))
common.DebugPrint("multi_scale_harris_Evangelidis(): points.shape = %s" %
str(points.shape))
tMSH2 = float(cv2.getTickCount())
myTime = (tMSH2 - tMSH1) / cv2.getTickFrequency()
print(
"multi_scale_harris_Evangelidis(): multi_scale_harris_Evangelidis() took %.6f [sec]"
% myTime)
return points