def TemporalAlignment(captureQ, captureR):
global harlocsQ, harlocsR
# if compute_harris == 1:...
totalT1 = float(cv2.getTickCount())
#if config.PREPROCESS_REFERENCE_VIDEO_ONLY == True: # This will give error in multiscale_quad_retrieval since we need to load the saved Harris features anyhow
if True:
# We compute and Store in files the multi-scale Harris features of the reference video
"""
harlocsR = ComputeHarlocs(captureR, config.counterRStep, \
folderName="/harlocs_ref", fileNamePrefix="harloc");
"""
harlocsR = ComputeHarlocs(captureR, config.counterRStep, \
folderName=config.HARRIS_REFERENCE_FOLDER_NAME, \
fileNamePrefix=config.HARRIS_FILENAME_PREFIX, \
fileNameExtension=config.HARRIS_FILENAME_EXTENSION,
indexVideo=1)
if common.MY_DEBUG_STDOUT:
common.DebugPrint("TemporalAlignment(): len(harlocsR) = %s" %
str(len(harlocsR)))
sumNbytes = 0
for hr in harlocsR:
sumNbytes += hr.nbytes
common.DebugPrint("TemporalAlignment(): harlocsR.nbytes = %s" %
str(sumNbytes))
#common.DebugPrint("TemporalAlignment(): harlocsR.nbytes = %s" % str(harlocsR.nbytes));
if config.PREPROCESS_REFERENCE_VIDEO_ONLY == False:
# We compute and Store in files the multi-scale Harris features of the query video
"""
Note: if the "harloc" files exist, load directly the features from
the files without computing them again.
"""
harlocsQ = ComputeHarlocs(captureQ, config.counterQStep, \
folderName=config.HARRIS_QUERY_FOLDER_NAME, \
fileNamePrefix=config.HARRIS_FILENAME_PREFIX, \
fileNameExtension=config.HARRIS_FILENAME_EXTENSION,
indexVideo=0)
if common.MY_DEBUG_STDOUT:
common.DebugPrint("TemporalAlignment(): len(harlocsQ) = %s" % \
str(len(harlocsQ)))
sumNbytes = 0
for hq in harlocsQ:
sumNbytes += hq.nbytes
common.DebugPrint("TemporalAlignment(): harlocsQ.nbytes = %s" %
str(sumNbytes))
#res = QuadTreeDecision(captureQ, captureR);
res = QuadTreeDecision()
else:
res = None
totalT2 = float(cv2.getTickCount())
myTime = (totalT2 - totalT1) / cv2.getTickFrequency()
print("TemporalAlignment() took %.6f [sec]" % (myTime))
return res
def process_input_frames(capture_q, capture_r, f_output):
# Allocate num_features_matched
num_features_matched = [None] * num_frames_q
for i in range(num_frames_q):
num_features_matched[i] = [-2000000000] * num_frames_r
while True:
if config.OCV_OLD_PY_BINDINGS:
frame_q = capture_q.get(cv2.cv.CV_CAP_PROP_POS_FRAMES)
else:
frame_q = capture_q.get(cv2.CAP_PROP_POS_FRAMES)
common.DebugPrint("Alex: frame_q = %d" % frame_q)
counter_q = int(frame_q)
common.DebugPrint("Alex: counter_q = %d" % counter_q)
ret1, imgQ = capture_q.read()
if False and config.SAVE_FRAMES:
file_name = config.IMAGES_FOLDER + "/imgQ_%05d.png" % counter_q
if not os.path.exists(file_name):
cv2.imwrite(file_name, imgQ)
if (ret1 == False) or (counter_q > num_frames_q):
break
# TODO: counter_q already visible in module MatchFrames
compute_features_1(imgQ, counter_q)
# We set the video stream capture_r at the beginning
if config.OCV_OLD_PY_BINDINGS:
capture_r.set(cv2.cv.CV_CAP_PROP_POS_FRAMES, 0)
else:
capture_r.set(cv2.CAP_PROP_POS_FRAMES, 0)
# Start time profiling for the inner loop
t1 = float(cv2.getTickCount())
# TODO: counter_q already visible in module MatchFrames
temporal_alignment(counter_q, frame_q, capture_r, num_frames_r,
num_features_matched, f_output)
# Measuring how much it takes the inner loop
t2 = float(cv2.getTickCount())
my_time = (t2 - t1) / cv2.getTickFrequency()
common.DebugPrint(
"Avg time it takes to complete a match (and to perform "
"INITIAL Feat-Extract) = %.6f [sec]" %
(my_time / (num_frames_r / config.counterRStep)))
counter_q += config.counterQStep
# If we try to seek to a frame out-of-bounds frame it gets to the
# last one
if config.OCV_OLD_PY_BINDINGS:
capture_q.set(cv2.cv.CV_CAP_PROP_POS_FRAMES, counter_q)
else:
capture_q.set(cv2.CAP_PROP_POS_FRAMES, counter_q)
common.DebugPrint("num_features_matched = %s" % str(num_features_matched))
def filter_keypoints_matches(kp1, kp2, matches, ratio=0.95):
global nonp1, nonp2
global myTemporalAlignmentCost
myTemporalAlignmentCost = 0
mkp1, mkp2 = [], []
nonp1, nonp2 = [], []
index = 0
for m in matches:
len_m = len(m)
if len_m == 2:
if m[0].distance < m[1].distance * ratio:
"""
Inspired from
http://www.cs.ubc.ca/~lowe/papers/ijcv04.pdf#page=20:
"A more effective measure is obtained by comparing the
distance of the closest neighbor to that of the
second-closest neighbor."
Note:
m[0] = closest neighbor match
m[1] = 2nd-closest neighbor match
"""
# These keypoints are considered matched -
# we choose the 1st closest match
mkp1.append(kp1[m[0].queryIdx])
mkp2.append(kp2[m[0].trainIdx])
myTemporalAlignmentCost += m[0].distance
else:
# These keypoints are considered NOT matched
nonp1.append(kp1[m[0].queryIdx].pt)
# TODO: I think that nonp2.append below is NOT relevant
else:
assert len_m <= 2
# These keypoints are considered NOT matched
if len_m == 1:
nonp1.append(kp1[m[0].queryIdx].pt)
else:
nonp1.append(kp1[index].pt)
index += 1
"""
nonp1 is a list of pairs of x, y coordinates for the features/keypoints
that have not been matched from frame of video A.
"""
common.DebugPrint("len(nonp1) = %d" % len(nonp1))
common.DebugPrint("len(nonp2) = %d" % len(nonp2))
p1 = np.float32([kp.pt for kp in mkp1])
p2 = np.float32([kp.pt for kp in mkp2])
kp_pairs = zip(mkp1, mkp2)
return p1, p2, kp_pairs
def onmouse(event, x, y, flags, param):
crt_vis = vis
if flags & cv2.EVENT_FLAG_LBUTTON:
crt_vis = vis0.copy()
r = 3
m = (common_cv.anorm(p1_local - (x, y)) < r) | \
(common_cv.anorm(p2_local - (x, y)) < r)
idxs = np.where(m)[0]
kp1s, kp2s = [], []
assert len(p1_local) == len(p2_local)
for i in idxs:
(x1, y1), (x2, y2) = p1_local[i], p2_local[i]
try:
col = (red, green)[status_local[i]]
except:
common.DebugPrintErrorTrace()
common.DebugPrint("onmouse() exception: i = %d, "
"len(statusLocal) = %d" %
(i, len(status_local)))
col = red
cv2.line(crt_vis, (x1, y1), (x2, y2), col)
try:
kp1, kp2 = kp_pairs[i]
kp1s.append(kp1)
kp2s.append(kp2)
except:
common.DebugPrintErrorTrace()
common.DebugPrint(
"onmouse() exception2: i = %d, "
"len(kp_pairs)=%d, len(idxs)=%d, idxs=%s" %
(i, len(kp_pairs), len(idxs), str(idxs)))
crt_vis = cv2.drawKeypoints(crt_vis,
kp1s,
flags=4,
color=kp_color)
crt_vis[:, w_q:] = cv2.drawKeypoints(crt_vis[:, w_q:],
kp2s,
flags=4,
color=kp_color)
cv2.imshow(win, crt_vis)
def compute_features_and_match_2(a_img_r, a_counter_r):
global kp2, desc2
global counter_r
global img_r
img_r = a_img_r
counter_r = a_counter_r
if counter_r is None:
counter_r = 0
common.DebugPrint(
"compute_features_and_match_2(): counter_r = %d (counter_q = %d)" %
(counter_r, counter_q))
# If we did NOT memoize kp2[counter_r], desc2[counter_r] we compute
# them once
if kp2[counter_r] is None:
"""
From http://docs.opencv.org/trunk/modules/nonfree/doc/feature_detection.html
"Python API provides three functions.
First one finds keypoints only.
Second function computes the descriptors based on the keypoints we
provide.
Third function detects the keypoints and computes their descriptors
If you want both keypoints and descriptors, directly use third
function as kp, des = cv2.SIFT.detectAndCompute(image, None)"
"""
t1 = float(cv2.getTickCount())
kp2[counter_r], desc2[counter_r] = detector.detectAndCompute(
a_img_r, None)
t2 = float(cv2.getTickCount())
my_time = (t2 - t1) / cv2.getTickFrequency()
common.DebugPrint("compute_features_and_match_2(): "
"detector.detectAndCompute() took %.6f [sec]" %
my_time)
common.DebugPrint("a_img_r - %d features" % len(kp2[counter_r]))
res = feature_match_and_homography()
"""
The result is related to the homography transformation returned by
feature_match_and_homography(): status sum and len
"""
return res
def IterationStandaloneMSH(index):
#capture = g.capture;
harlocsFolder = g.harlocsFolder
fileNamePrefix = g.fileNamePrefix
fileNameExtension = g.fileNameExtension
common.DebugPrint("Entered IterationStandaloneMSH(index=%d)" % index)
#img = MyImageReadMSH(capture, index);
img = MyImageReadMSH(index)
im = img
pp = multi_scale_harris.multi_scale_harris(im, nos, disp=0)
# n=0:nos-1
if False:
#harlocs = pp
harlocs.append(pp)
multi_scale_harris.StoreMultiScaleHarrisFeatures( \
harlocsFolder + "/" + fileNamePrefix + "%05d%s" % \
(index, fileNameExtension),
pp)
if False:
counter += counterStep
# If we try to seek to a frame out-of-bounds frame it gets to the last one
if config.OCV_OLD_PY_BINDINGS:
capture.set(cv2.cv.CV_CAP_PROP_POS_FRAMES, counter)
else:
capture.set(cv2.CAP_PROP_POS_FRAMES, counter)
return 1
def rest(win):
global nonp1
global status
# Here we perform spatial alignment and clustering and display the results.
t1 = float(cv2.getTickCount())
# TODO: filter out p1 outside the alignment of the pair of frames
(myVis, inputFrame, refFrame,
status2) = spatial_alignment(win, img_q, img_r, kp_pairs, status, H)
t2 = float(cv2.getTickCount())
my_time = (t2 - t1) / cv2.getTickFrequency()
common.DebugPrint(
"feature_match_and_homography(): spatial_alignment() took %.6f [sec]" %
my_time)
# TODO: filter out nonp1 outside the alignment of the pair of frames -
# should do it on p1 inside spatial_alignment()
# We cluster the non-matched keypoints
nonp1 = cluster_unmatched_keypoints(nonp1)
my_vis2 = annotate_vis(win, myVis, inputFrame, refFrame, status)
# TODO: remove global definition of vis
global vis
vis = my_vis2
if config.USE_GUI:
ch = cv2.waitKey()
print("ch = %s" % str(ch))
if ch == 27: # Escape key
quit()
cv2.destroyAllWindows()
def write_video_capture(video_file_name, folder_name):
# OpenCV can read only AVIs - not 3GP, nor FLVs with MPEG compression
# From http://answers.opencv.org/question/6/how-to-readwrite-video-with-opencv-in-python/
video_writer = None
folder_content = os.listdir(folder_name)
sorted_folder_content = sorted(folder_content)
for fileName in sorted_folder_content:
path_file_name = folder_name + "/" + fileName
if os.path.isfile(path_file_name) and \
fileName.lower().endswith("_good.png"):
common.DebugPrint("ComputeHarlocs(): Loading %s" % path_file_name)
img = cv2.imread(path_file_name)
assert img is not None
if video_writer:
common.DebugPrint("img.shape = %s" % str(img.shape))
# From http://docs.opencv.org/trunk/doc/py_tutorials/py_gui/py_video_display/py_video_display.html#saving-a-video
# See also http://opencv-python-tutroals.readthedocs.org/en/latest/py_tutorials/py_gui/py_video_display/py_video_display.html
# WRITES 0 BYTES IN THE VIDEO: vid_fourcc = cv2.VideoWriter_fourcc('M','J','P','G');
# See also http://www.fourcc.org/codecs.php
vid_fourcc = cv2.VideoWriter_fourcc(*'XVID')
video_writer = cv2.VideoWriter(filename=video_file_name,
fourcc=vid_fourcc,
fps=10,
frameSize=(img.shape[1],
img.shape[0]))
else:
common.DebugPrint("Error in creating video writer")
sys.exit(1)
video_writer.write(img)
video_writer.release()
common.DebugPrint("Finished writing the video")
return
def open_video_capture(video_file_name, video_type):
"""
Unfortunately, normally cv2.VideoCapture() continues even if it does not
find videoPathFileName
"""
assert os.path.isfile(video_file_name)
# From http://docs.opencv.org/modules/highgui/doc/reading_and_writing_images_and_video.html#videocapture-videocapture
capture = cv2.VideoCapture(video_file_name)
# Inspired from https://stackoverflow.com/questions/16703345/how-can-i-use-opencv-python-to-read-a-video-file-without-looping-mac-os
if config.OCV_OLD_PY_BINDINGS:
frame_count = int(capture.get(cv2.cv.CV_CAP_PROP_FRAME_COUNT))
else:
frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))
if config.OCV_OLD_PY_BINDINGS:
capture.set(cv2.cv.CV_CAP_PROP_POS_FRAMES,
config.initFrame[video_type])
else:
capture.set(cv2.CAP_PROP_POS_FRAMES, config.initFrame[video_type])
if config.OCV_OLD_PY_BINDINGS:
width = capture.get(cv2.cv.CV_CAP_PROP_FRAME_WIDTH)
height = capture.get(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT)
fps = capture.get(cv2.cv.CV_CAP_PROP_FPS)
codec = capture.get(cv2.cv.CV_CAP_PROP_FOURCC)
else:
width = capture.get(cv2.CAP_PROP_FRAME_WIDTH)
height = capture.get(cv2.CAP_PROP_FRAME_HEIGHT)
fps = capture.get(cv2.CAP_PROP_FPS)
codec = capture.get(cv2.CAP_PROP_FPS)
assert width < 32767 # we use np.int16
assert height < 32767 # we use np.int16
duration = frame_count / fps
print("Video '%s' has resolution %dx%d, %.2f fps and "
"%d frames, duration %.2f secs, codec=%s" %
(video_file_name, width, height, fps, frame_count, duration, codec))
steps = [config.counterQStep, config.counterRStep]
# TODO: take into account also initFrame
used_frame_count = frame_count / steps[video_type]
assert not ((video_type == 0) and (used_frame_count <= 10))
common.DebugPrint(
"We use video '%s', with %d frames, from which we use ONLY %d\n" %
(video_file_name, frame_count, used_frame_count))
resolution = (width, height)
return capture, frame_count, resolution
def testSpatialAlignmentEvangelidis(self):
"""
We test Evangelidis Spatial Alignment, given the precomputed
temporal alignment, the cross array.
"""
videoPathFileNameQ = "Videos/input.avi"
videoPathFileNameR = "Videos/reference.avi"
"""
We want to use the same JPEGs Matlab is using - since the algorithm
is somewhat sensitive to quantisation errors.
"""
assert config.TESTING_IDENTICAL_MATLAB == True
import ReadVideo
captureQ, frameCountQ, resVidQ = ReadVideo.OpenVideoCapture(
videoPathFileNameQ, 0)
common.DebugPrint("Alex: frameCountQ = %d" % frameCountQ)
captureR, frameCountR, resVidR = ReadVideo.OpenVideoCapture(
videoPathFileNameR, 1)
common.DebugPrint("Alex: frameCountR = %d" % frameCountR)
"""
The cross result we obtain for the videos from
Evangelidis, with step=25 (1fps).
"""
crossref = [[0, 48], [1, 48], [2, 48], [3, 48], [4, 48], [5, 48],
[6, 0], [7, 0], [8, 60], [9, 60], [10, 60], [11, 60],
[12, 67], [13, 67], [14, 67], [15, 67], [16, 67], [17, 67],
[18, 72], [19, 72], [20, 72], [21, 78], [22, 78], [23, 78],
[24, 78], [25, 78], [26, 82], [27, 82], [28, 82], [29, 54],
[30, 54]]
crossref = np.array(crossref)
#SpatialAlignmentEvangelidis(cross, captureQ=None, captureR=None);
res = SpatialAlignmentEvangelidis(crossref, captureQ, captureR)
common.DebugPrint("Corrected cross from SpatialAlignmentEvangelidis() = %s" % \
str(res))
"""
def spatial_consistency(space_xy, qcen, M, st_threshold, cropflag):
ordvot = np.zeros((M, 1))
for iii in range(1, M + 1):
#uu=space_xy(:,(iii-1)*2+1:(iii-1)*2+2);
uu = space_xy[:, (iii - 1) * 2:(iii - 1) * 2 + 1]
#pn=~isnan(uu(:,1));
pn = np.logical_not(np.isnan(uu[:, 0]))
if isempty(pn):
continue
#vv=(uu(pn,:)-qcen(pn,:));
vv = (uu[pn, :] - qcen[pn, :])
if cropflag == 1:
#inliers=(sign(vv(:,2))==-1)&(abs(vv(:,1))<10);
inliers = (sign(vv[:, 1]) == -1) and (abs(vv[:, 0]) < 10)
#outliers = (sign(vv(:,2))==1)|(abs(vv(:,1))>10)
outliers = (sign(vv[:, 1]) == 1) or (abs(vv[:, 0]) > 10)
bifla = sum(inliers)
#/(sum(outliers)+sum(inliers))
if isnan(bifla):
ordvot[iii - 1] = 0
elif isinf(bifla):
ordvot[iii - 1] = inliers.sum()
else:
ordvot[iii - 1] = bifla
else:
#inliers=sqrt(sum(vv.^2,2))<st_threshold;
inliers = sqrt((vv**2).sum(1)) < st_threshold
#outliers = sqrt(sum(vv.^2,2)) > st_threshold
outliers = sqrt((vv**2).sum(1)) > st_threshold
bifla = inliers.sum() / (outliers.sum() + inliers.sum())
if isnan(bifla):
ordvot[iii - 1] = 0
elif isinf(bifla):
ordvot[iii - 1] = inliers.sum()
else:
ordvot[iii - 1] = bifla
if common.MY_DEBUG_STDOUT:
common.DebugPrint("ordvot = %s" % str(ordvot))
return ordvot
def simulated_annealing():
L = GetALength()
common.DebugPrint("simulated_annealing(): L = %s" % str(L))
#L = len(A)
x0 = random.choice(xrange(0, L))
T = 1.
k = 1
x = x0
x_best = x0
# Alex:
#while T > 1e-3:
#while T > 1e-1: # 901 iterations
#while T > 1e-2: # 991 iterations
#while T > 5e-1: # 501 iterations
#while T > 6e-1: # 401 iterations
#while T > 7e-1: # 301 iterations
#while T > 8e-1: # 201 iterations
#while T > 9.5e-1: # 51 iterations - suboptimal (401, 228) instead of (400, 377)
#while T > 9.7e-1: # 31 iterations - suboptimal (402, 198) instead of (400, 377)
"""
In a run obtained suboptimal (411, 78) instead of (400, 377), worse than 51 iterations
In another run obtained optimal (400, 377) result.
For our example from test.py this one converges nice 50% of the times.
"""
while T > 9e-1: # 101 iterations:
x = make_move(x, T)
if GetA(x) < GetA(x_best):
x_best = x
T = update_temperature(T, k)
k += 1
common.DebugPrint("iterations: %d" % k)
return x, x_best, x0
def Benchmark():
global captureQ, frameCountQ
global captureR, frameCountR
# Doing benchmarking
while True:
if config.OCV_OLD_PY_BINDINGS:
frame1 = captureQ.get(cv2.cv.CV_CAP_PROP_POS_FRAMES)
else:
frame1 = captureQ.get(cv2.CAP_PROP_POS_FRAMES)
common.DebugPrint("Alex: frame1 = %d" % frame1)
MatchFrames.counterQ = int(frame1) #0
common.DebugPrint("Alex: MatchFrames.counterQ = %d" %
MatchFrames.counterQ)
retQ, imgQ = captureQ.read()
if retQ == False:
break
if False:
grayQ = common.ConvertImgToGrayscale(imgQ)
MatchFrames.Main_img1(imgQ, MatchFrames.counterQ)
# 36.2 secs (38.5 secs with Convert to RGB)
common.DebugPrint("Alex: time after Feature Extraction of all frames of " \
"video 1 = %s" % GetCurrentDateTimeStringWithMilliseconds())
while True:
if config.OCV_OLD_PY_BINDINGS:
frameR = captureR.get(cv2.cv.CV_CAP_PROP_POS_FRAMES)
else:
frameR = captureR.get(cv2.CAP_PROP_POS_FRAMES)
common.DebugPrint("Alex: frameR = %d" % frameR)
MatchFrames.counterR = int(frameR)
#0
common.DebugPrint("Alex: counterR = %d" % (MatchFrames.counterR))
retR, imgR = captureR.read()
if retR == False:
break
if False:
grayR = common.ConvertImgToGrayscale(imgR)
MatchFrames.Main_img2(imgR, MatchFrames.counterR)
# Note: 47.2 secs (56.7 secs with Convert to RGB)
common.DebugPrint("Alex: time after Feature Extraction of all frames of " \
"video 2 and (FLANN?) matching once for each frame = %s" % \
GetCurrentDateTimeStringWithMilliseconds())
quit()
def compute_features_1(a_img_1, a_counter_q):
global kp1, desc1
global counter_q
global img_q
img_q = a_img_1
counter_q = a_counter_q
common.DebugPrint("compute_features_1(): counter_q = %d" % counter_q)
if kp1[counter_q] is None:
t1 = float(cv2.getTickCount())
kp1[counter_q], desc1[counter_q] = detector.detectAndCompute(
a_img_1, None)
t2 = float(cv2.getTickCount())
my_time = (t2 - t1) / cv2.getTickFrequency()
common.DebugPrint(
"compute_features_1(): detector.detectAndCompute() took %.6f [sec]"
% my_time)
common.DebugPrint("a_img_1 - %d features" % len(kp1[counter_q]))
def cluster_unmatched_keypoints(z):
# Start time profiling for the inner loop
t1 = float(cv2.getTickCount())
z = np.array(z)
# convert to np.float32
z = np.float32(z)
n = len(z) # the number of points
common.DebugPrint("N = %d" % n)
# TODO: should we cluster also the non-matched features of the reference
# video (B) ?
# TODO:_IMPORTANT: now Hierarchical Clustering returns the clusters -
# you can color them differently
z = Clustering.hierarchical_clustering(z, n)
t2 = float(cv2.getTickCount())
my_time = (t2 - t1) / cv2.getTickFrequency()
common.DebugPrint(
"cluster_unmatched_keypoints(): HierarchicalClustering() "
"took %.5f [sec]" % my_time)
return z
def benchmark():
global captureQ, frameCountQ
global captureR, frameCountR
# Doing benchmarking
while True:
if config.OCV_OLD_PY_BINDINGS:
frame1 = captureQ.get(cv2.cv.CV_CAP_PROP_POS_FRAMES)
else:
frame1 = captureQ.get(cv2.CAP_PROP_POS_FRAMES)
common.DebugPrint("Alex: frame1 = %d" % frame1)
MatchFrames.counter_q = int(frame1)
common.DebugPrint("Alex: MatchFrames.counter_q = %d" %
MatchFrames.counter_q)
ret_q, img_q = captureQ.read()
if not ret_q:
break
# TODO: This method below doesn't seem to exist?
# MatchFrames.Main_img1(img_q, MatchFrames.counter_q)
# 36.2 secs (38.5 secs with Convert to RGB)
common.DebugPrint("Alex: time after Feature Extraction of all frames of "
"video 1 = %s" %
common.GetCurrentDateTimeStringWithMilliseconds())
while True:
if config.OCV_OLD_PY_BINDINGS:
frame_r = captureR.get(cv2.cv.CV_CAP_PROP_POS_FRAMES)
else:
frame_r = captureR.get(cv2.CAP_PROP_POS_FRAMES)
common.DebugPrint("Alex: frame_r = %d" % frame_r)
MatchFrames.counter_r = int(frame_r)
common.DebugPrint("Alex: counter_r = %d" % MatchFrames.counter_r)
ret_r, img_r = captureR.read()
if not ret_r:
break
# TODO: This method below doesn't seem to exist?
# MatchFrames.Main_img2(img_r, MatchFrames.counter_r)
# Note: 47.2 secs (56.7 secs with Convert to RGB)
common.DebugPrint(
"Alex: time after Feature Extraction of all frames of "
"video 2 and (FLANN?) matching once for each frame = %s" %
common.GetCurrentDateTimeStringWithMilliseconds())
quit()
def GetA(aCounter2):
global capture2
# We get the frame at each access and convert it to RGB - we don't store the result in memory
# We memoize the value corresponding to aCounter2 in Acache[aCounter2], in the idea that Simulated Annealing might come back to this value
if aCounter2 in Acache:
return Acache[aCounter2]
# If we try to seek to a frame out-of-bounds frame it gets to the last one
capture2.set(cv2.cv.CV_CAP_PROP_POS_FRAMES, aCounter2)
frame2 = capture2.get(cv2.cv.CV_CAP_PROP_POS_FRAMES)
if False:
print "Alex: frame2 = %d" % frame2
#aCounter2 = int(frame2) #0
assert aCounter2 == int(frame2) #0
ret2, img2 = capture2.read()
if (ret2 == False) or (GetALength() <= aCounter2):
common.DebugPrint(
"Get(): REALLY BAD error: aCounter2 = %d, ret2 = %s" % \
(aCounter2, str(ret2)))
quit()
#break
if False:
# Note: img1 and img2 are RGB-images. I need to convert them to grayscale (as cv2.imread(fn1, 0) )
gray2 = ConvertImgToGrayscale(img2)
res = MatchFrames.Main_img2(img2, aCounter2)
#print "GetA(): res = %s" % (str(res))
# We memoize this result - The ReadVideo module is responsible to reset this dictionary for a new processing
Acache[aCounter2] = -res[1]
# We return the number of matches
return -res[1]
def pre_main(n_frames_q, n_frames_r):
global detector, matcher
global num_frames_q, num_frames_r
global kp1, kp2
global desc1, desc2
"""
# From http://docs.scipy.org/doc/numpy/reference/generated/numpy.set_printoptions.html
numpy.set_printoptions(precision=None, threshold=None, edgeitems=None,
linewidth=None, suppress=None, nanstr=None,
infstr=None, formatter=None)
threshold : int, optional
Total number of array elements which trigger summarization rather
than full repr (default 1000).
suppress : bool, optional
Whether or not suppress printing of small floating point values
using scientific notation (default False).
"""
np.set_printoptions(threshold=1000000, linewidth=3000)
num_frames_q = n_frames_q
num_frames_r = n_frames_r
feature_name = config.FEATURE_DETECTOR_AND_MATCHER
common.DebugPrint("Alex: feature_name = %s" % feature_name)
detector, matcher = init_feature(feature_name)
if detector is not None:
print("using %s" % feature_name)
else:
print("unknown feature: %s" % feature_name)
sys.exit(1)
kp1 = [None] * num_frames_q
kp2 = [None] * num_frames_r
desc1 = [None] * num_frames_q
desc2 = [None] * num_frames_r
scipy.show_config();
# See http://docs.scipy.org/doc/numpy/reference/generated/numpy.set_printoptions.html
# We use 7 digits precision and suppress using scientific notation.
np.set_printoptions(precision=7, suppress=True, \
threshold=70000, linewidth=4000);
#threshold=7000000, linewidth=4000);
#threshold=7000, linewidth=300);
#threshold=1000000, linewidth=3000);
# Inspired from \OpenCV2-Python-Tutorials-master\source\py_tutorials\py_core\py_optimization
# normally returns True - relates to using the SIMD extensions of x86: SSX, AVX
common.DebugPrint("cv2.useOptimized() is %s" % str(cv2.useOptimized()));
if False:
cv2.setUseOptimized(True);
cv2.useOptimized();
"""
From http://docs.opencv.org/modules/core/doc/utility_and_system_functions_and_macros.html#checkhardwaresupport
CV_CPU_MMX - MMX
CV_CPU_SSE - SSE
CV_CPU_SSE2 - SSE 2
CV_CPU_SSE3 - SSE 3
CV_CPU_SSSE3 - SSSE 3
CV_CPU_SSE4_1 - SSE 4.1
CV_CPU_SSE4_2 - SSE 4.2
CV_CPU_POPCNT - POPCOUNT
def multiscale_quad_tree(r_path, threshold, scale_index):
common.DebugPrint("Entered multiscale_quad_tree(scale_index=%d)" %
scale_index)
t1 = float(cv2.getTickCount())
found_files = False
try:
all_quads = np.load("all_quads%d.npz" % scale_index)['arr_0']
all_id = np.load("all_id%d.npz" % scale_index)['arr_0']
all_cen = np.load("all_cen%d.npz" % scale_index)['arr_0']
all_max = np.load("all_max%d.npz" % scale_index)['arr_0']
all_ori = np.load("all_ori%d.npz" % scale_index)['arr_0']
n_d = np.load("n_d%d.npz" % scale_index)['arr_0']
found_files = True
common.DebugPrint("multiscale_quad_tree(): loaded the NPZ files for "
"this scale.")
print(
"multiscale_quad_tree(scale_index=%d): time before starting the "
"processing for the given query video = %s" % scale_index,
common.GetCurrentDateTimeStringWithMilliseconds())
except:
common.DebugPrintErrorTrace()
if not found_files:
rd = r_path
# OpenCV's KD-tree implementation does NOT accept float64
all_quads = np.array([]).astype(np.float32)
all_cen = np.array([]).astype(np.float32)
all_id = np.array([]).astype(np.float32)
all_max = np.array([]).astype(np.float32)
all_ori = np.array([]).astype(np.float32)
n_d = np.zeros((len(rd), 1))
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_tree(): n_d.shape = "
"%s" % str(n_d.shape))
common.DebugPrint("multiscale_quad_tree(): n_d = %s" % str(n_d))
# Alex: for each reference video frame we compute the quads
for iFor in range(len(rd)):
# Alex: IMPORTANT: This loads into pp the multiscale Harris feature
# saved in file r_path+rd(i).name
# load harris locations of image (already computed)
pp = r_path[iFor]
if iFor % 10 == 0:
common.DebugPrint("multiscale_quad_tree(): iFor = %d" % iFor)
common.DebugPrint("multiscale_quad_tree(): scale_index = "
"%s" % str(scale_index))
common.DebugPrint("multiscale_quad_tree(): threshold = "
"%s" % str(threshold))
# Alex: We put in points the rows of pp having the
# 3rd element == scale_index, and we take out the 3rd column of pp
# [out,cen,maxdis,ori]=findquads(pp(pp(:,3)==scale_index,1:2),threshold,1)
points = pp[pp[:, 2] == scale_index, 0:2]
out, cen, maxdis, ori = findquads.findquads(points, threshold, 1)
n_d[iFor] = out.shape[0]
temp = np.zeros((out.shape[0], 1)) + iFor
if common.MY_DEBUG_STDOUT:
common.DebugPrint("Initially:")
common.DebugPrint(
" multiscale_quad_tree(): all_quads.shape = %s" %
str(all_quads.shape))
common.DebugPrint(" multiscale_quad_tree(): all_quads = %s" %
str(all_quads))
common.DebugPrint(
" multiscale_quad_tree(): all_max.shape = %s" %
str(all_max.shape))
common.DebugPrint(" multiscale_quad_tree(): all_max = %s" %
str(all_max))
common.DebugPrint(
" multiscale_quad_tree(): maxdis.shape = %s" %
str(maxdis.shape))
common.DebugPrint(" multiscale_quad_tree(): maxdis = %s" %
str(maxdis))
common.DebugPrint(
" multiscale_quad_tree(): all_ori.shape = %s" %
str(all_ori.shape))
common.DebugPrint(" multiscale_quad_tree(): ori.shape = %s" %
str(ori.shape))
common.DebugPrint(" multiscale_quad_tree(): ori = %s" %
str(ori))
common.DebugPrint(
" multiscale_quad_tree(): all_cen.shape = %s" %
str(all_cen.shape))
common.DebugPrint(" multiscale_quad_tree(): all_cen = %s" %
str(all_cen))
common.DebugPrint(" multiscale_quad_tree(): cen.shape = %s" %
str(cen.shape))
common.DebugPrint(" multiscale_quad_tree(): cen = %s" %
str(cen))
common.DebugPrint(" multiscale_quad_tree(): out.shape = %s" %
str(out.shape))
common.DebugPrint(" multiscale_quad_tree(): out = %s" %
str(out))
if out.size == 0:
assert (cen.size == 0)
assert (maxdis.size == 0)
assert (ori.size == 0)
continue
"""
It crashes at
all_quads = np.r_[all_quads, out]
with "ValueError: array dimensions must agree except for d_0"
because:
multiscale_quad_tree(): out = []
multiscale_quad_tree(): out.shape = (2, 0)
"""
if all_quads.size == 0:
all_quads = out.copy()
else:
all_quads = np.r_[all_quads, out]
if all_cen.size == 0:
all_cen = cen.copy()
else:
all_cen = np.r_[all_cen, cen]
if all_id.size == 0:
all_id = temp.copy()
else:
all_id = np.r_[all_id, temp]
if all_max.size == 0:
all_max = maxdis.copy()
else:
all_max = np.r_[all_max, maxdis]
if all_ori.size == 0:
all_ori = ori.copy()
else:
all_ori = np.r_[all_ori, ori]
try:
np.savez_compressed("all_quads%d" % scale_index, all_quads)
np.savez_compressed("all_id%d" % scale_index, all_id)
np.savez_compressed("all_cen%d" % scale_index, all_cen)
np.savez_compressed("all_max%d" % scale_index, all_max)
np.savez_compressed("all_ori%d" % scale_index, all_ori)
np.savez_compressed("n_d%d" % scale_index, n_d)
except:
common.DebugPrintErrorTrace()
if all_quads.size == 0:
t2 = float(cv2.getTickCount())
my_time = (t2 - t1) / cv2.getTickFrequency()
common.DebugPrint("multiscale_quad_tree() "
"took %.6f [sec]" % my_time)
return None, all_id, all_cen, all_max, all_ori, n_d, all_quads
t_build1 = float(cv2.getTickCount())
if config.KDTREE_IMPLEMENTATION == 0:
tree = spatial.KDTree(all_quads)
elif config.KDTREE_IMPLEMENTATION == 1:
# TODO: try to use exact NN-search for the kd-tree -
# see http://docs.opencv.org/trunk/modules/flann/doc/flann_fast_approximate_nearest_neighbor_search.html
all_quads = all_quads.astype(np.float32)
tree = cv2.flann_Index(features=all_quads, params=config.FLANN_PARAMS)
t_build2 = float(cv2.getTickCount())
my_time_build = (t_build2 - t_build1) / cv2.getTickFrequency()
print("multiscale_quad_tree(): KD-tree build "
"took %.6f [sec]" % my_time_build)
if common.MY_DEBUG_STDOUT:
common.DebugPrint("At the end:")
common.DebugPrint(" multiscale_quad_tree(): all_id.shape = %s" %
str(all_id.shape))
common.DebugPrint(" multiscale_quad_tree(): all_id = %s" %
str(all_id))
common.DebugPrint(" multiscale_quad_tree(): all_cen.shape = %s" %
str(all_cen.shape))
common.DebugPrint(" multiscale_quad_tree(): all_cen = %s" %
str(all_cen))
common.DebugPrint(" multiscale_quad_tree(): all_max.shape = %s" %
str(all_max.shape))
common.DebugPrint(" multiscale_quad_tree(): all_max = %s" %
str(all_max))
common.DebugPrint(" multiscale_quad_tree(): all_ori.shape = %s" %
str(all_ori.shape))
common.DebugPrint(" multiscale_quad_tree(): all_ori = %s" %
str(all_ori))
common.DebugPrint(" multiscale_quad_tree(): n_d.shape = %s" %
str(n_d.shape))
common.DebugPrint(" multiscale_quad_tree(): n_d = %s" % str(n_d))
common.DebugPrint(" multiscale_quad_tree(): all_quads.shape = %s" %
str(all_quads.shape))
common.DebugPrint(" multiscale_quad_tree(): all_quads = %s" %
str(all_quads))
common.DebugPrint(" multiscale_quad_tree(): all_quads.shape before "
"kd-tree = %s" % str(all_quads.shape))
try:
common.DebugPrint(
"multiscale_quad_tree(): sys.getsizeof(tree) = %s" %
str(sys.getsizeof(tree)))
except:
pass
common.DebugPrintErrorTrace()
t2 = float(cv2.getTickCount())
my_time = (t2 - t1) / cv2.getTickFrequency()
print("multiscale_quad_tree() took %.6f [sec]" % my_time)
return tree, all_id, all_cen, all_max, all_ori, n_d, all_quads
def main(argv):
assert len(sys.argv) >= 3
if FLAGS.preprocess_ref:
config.PREPROCESS_REFERENCE_VIDEO_ONLY = True
elif FLAGS.process_query_and_align_videos:
config.PREPROCESS_REFERENCE_VIDEO_ONLY = False
else:
config.PREPROCESS_REFERENCE_VIDEO_ONLY = False
print("config.PREPROCESS_REFERENCE_VIDEO_ONLY = %s" % str(
config.PREPROCESS_REFERENCE_VIDEO_ONLY))
ask_first()
# Inspired from https://stackoverflow.com/questions/1520234/how-to-check-which-version-of-numpy-im-using
print("numpy.version.version = %s" % str(np.version.version))
print("scipy.version.version = %s" % str(scipy.version.version))
np.show_config()
scipy.show_config()
# See http://docs.scipy.org/doc/numpy/reference/generated/numpy.set_printoptions.html
# We use 7 digits precision and suppress using scientific notation.
np.set_printoptions(precision=7, suppress=True,
threshold=70000, linewidth=4000)
# Inspired from \OpenCV2-Python-Tutorials-master\source\py_tutorials\py_core\py_optimization
# normally returns True - relates to using the SIMD extensions of x86:
# SSX, AVX
common.DebugPrint("cv2.useOptimized() is %s" % str(cv2.useOptimized()))
"""
From http://docs.opencv.org/modules/core/doc/utility_and_system_functions_and_macros.html#checkhardwaresupport
CV_CPU_MMX - MMX
CV_CPU_SSE - SSE
CV_CPU_SSE2 - SSE 2
CV_CPU_SSE3 - SSE 3
CV_CPU_SSSE3 - SSSE 3
CV_CPU_SSE4_1 - SSE 4.1
CV_CPU_SSE4_2 - SSE 4.2
CV_CPU_POPCNT - POPCOUNT
CV_CPU_AVX - AVX
"""
# TODO: Figure out the correct way to reference these in OpenCV 3.x
"""
# Need to setUseOptimized before calling checkHardwareSupport
cv2.setUseOptimized(True)
if config.OCV_OLD_PY_BINDINGS == False:
featDict = {cv2.CpuFeatures.CV_CPU_AVX: "AVX",
cv2.CPU_MMX: "MMX",
cv2.CPU_NEON: "NEON",
cv2.CPU_POPCNT: "POPCNT",
cv2.CPU_SSE: "SSE",
cv2.CPU_SSE2: "SSE2",
cv2.CPU_SSE3: "SSE3",
cv2.CPU_SSE4_1: "SSE4.1",
cv2.CPU_SSE4_2: "SSE4.2",
cv2.CPU_SSSE3: "SSSE3"}
for feat in featDict:
res = cv2.checkHardwareSupport(feat)
print("%s = %d" % (featDict[feat], res))
"""
# "Returns the number of logical CPUs available for the process."
common.DebugPrint("cv2.getNumberOfCPUs() (#logical CPUs) is %s" % str(
cv2.getNumberOfCPUs()))
common.DebugPrint(
"cv2.getTickFrequency() is %s" % str(cv2.getTickFrequency()))
video_file_q = sys.argv[1] # input/current video
video_file_r = sys.argv[2] # reference video
# TODO: use getopt() to run Evangelidis' or "Alex's" algorithm, etc
ReadVideo.main(video_file_q, video_file_r)
def MyImageReadMSH(index):
global g
"""
common.DebugPrint("MyImageReadMSH(): initFrame = %d)" % \
(config.initFrame[g.indexVideo]));
index += config.initFrame[g.indexVideo];
"""
"""
common.DebugPrint("Entered MyImageReadMSH(capture=%s, index=%s)" % \
(str(capture), str(index)));
"""
common.DebugPrint("Entered MyImageReadMSH(index=%s)" % \
(str(index)))
"""
We must reopen the capture device in each different process, otherwise the
program blocks at the first operation in the "global" capture device.
"""
if g.harlocsFolder.endswith(config.HARRIS_QUERY_FOLDER_NAME):
if g.captureQ == None:
capture = cv2.VideoCapture(sys.argv[1])
g.captureQ = capture
common.DebugPrint("MyImageReadMSH(): new capture=%s" % \
(str(capture)))
else:
capture = g.captureQ
elif g.harlocsFolder.endswith(config.HARRIS_REFERENCE_FOLDER_NAME):
if g.captureR == None:
capture = cv2.VideoCapture(sys.argv[2])
g.captureR = capture
common.DebugPrint("MyImageReadMSH(): new capture=%s" % \
(str(capture)))
else:
capture = g.captureR
else:
assert False
assert (g.indexVideo == 0) or (g.indexVideo == 1)
if config.OCV_OLD_PY_BINDINGS:
capture.set(cv2.cv.CV_CAP_PROP_POS_FRAMES, \
index)
else:
"""
From http://docs.opencv.org/modules/highgui/doc/reading_and_writing_images_and_video.html#videocapture-get:
<<CV_CAP_PROP_POS_FRAMES 0-based index of the frame to be
decoded/captured next.>>
"""
capture.set(cv2.CAP_PROP_POS_FRAMES, \
index)
common.DebugPrint("MyImageReadMSH(): after capture.set()")
# This is only for (paranoid) testing purposes:
if config.OCV_OLD_PY_BINDINGS:
indexCrt = capture.get(cv2.cv.CV_CAP_PROP_POS_FRAMES)
else:
"""
From http://docs.opencv.org/modules/highgui/doc/reading_and_writing_images_and_video.html#videocapture-get:
<<CV_CAP_PROP_POS_FRAMES 0-based index of the frame to be
decoded/captured next.>>
"""
indexCrt = capture.get(cv2.CAP_PROP_POS_FRAMES)
#assert int(indexCrt) == index;
#!!!!TODO: think if OK
if int(indexCrt) != index:
common.DebugPrint( \
"MyImageReadMSH(): indexCrt != index --> returning black frame")
ret = False
else:
#common.DebugPrint("Alex: frameR = %d" % frameR);
#if myIndex > numFramesR:
# break;
#ret, img = r_path.read();
ret, img = capture.read()
#if ret == False:
# break;
#!!!!TODO: think if well
#assert ret == True;
if ret == False:
common.DebugPrint(
"MyImageReadMSH(index=%d): ret == False --> returning None" %
index)
img = None
else:
common.DebugPrint("MyImageReadMSH(): img.shape = %s" % str(img.shape))
common.DebugPrint("MyImageReadMSH(): img.dtype = %s" % str(img.dtype))
#!!!!TODO: I suggest to do the gray conversion at reading, not in multi_scale_harris.py
if False:
# In the Matlab code he reads gray/8bpp JPEGs
imgGray = common.ConvertImgToGrayscale(img)
if config.VIDEO_FRAME_RESIZE_SCALING_FACTOR != 1:
# We resize the image
img = Matlab.imresize(img, \
scale=config.VIDEO_FRAME_RESIZE_SCALING_FACTOR)
common.DebugPrint("Exiting MyImageReadMSH()")
if False:
return imgGray
return img
def testSynchro(self):
common.MY_DEBUG_STDOUT = True
multiscale_quad_retrieval.DBGPRINT = True
#assert config.temporalDecisionType == 1; #1
"""
# 0 --> global solution (multi-scale dynamic programming - the dp3() function)
# 1 --> causal (local) syncronization
"""
if True: # We use dp_Alex (instead of causal, normally)
config.temporalDecisionType = 0
#dp3Orig = dp3;
multiscale_synchro_decision.dp3 = multiscale_synchro_decision.dp_Alex
config.KDTREE_IMPLEMENTATION = 1
# Use OpenCV's KDtree
config.FLANN_PARAMS = config.FLANN_PARAMS_DETERMINISTIC
"""
The cross result from Matlab, for the videos from
Evangelidis, with step=25 (1fps), when
applying the ORIGINAL causal() temporal alignment.
"""
resCrossref = [ \
[1001, 2049],
[1002, 2049],
[1003, 2049],
[1004, 2049],
[1005, 2049],
[1006, 2049],
[1007, 2001],
[1008, 2001],
[1009, 2061],
[1010, 2061],
[1011, 2061],
[1012, 2061],
[1013, 2068],
[1014, 2068],
[1015, 2068],
[1016, 2068],
[1017, 2068],
[1018, 2068],
[1019, 2073],
[1020, 2073],
[1021, 2073],
[1022, 2079],
[1023, 2079],
[1024, 2079],
[1025, 2079],
[1026, 2079],
[1027, 2083],
[1028, 2083],
[1029, 2083],
[1030, 2055],
[1031, 2055]]
resCrossref = np.array(resCrossref)
resCrossref[:, 0] -= 1001
resCrossref[:, 1] -= 2001
#common.DebugPrint("resCrossref = %s" % str(resCrossref));
#crossref = TemporalAlignment(None, None);
#crossref = QuadTreeDecision(None, None);
crossref = QuadTreeDecision()
common.DebugPrint("crossref = %s" % str(crossref))
aZero = resCrossref - crossref
common.DebugPrint("aZero = %s" % str(aZero))
"""
Note: if we use the OpenCV KD-tree implementation in findquads
(instead of SciPy implementation) we will have
different results on rows 5 and 6.
"""
if config.KDTREE_IMPLEMENTATION == 1:
common.DebugPrint("The following 2 values we disregard:")
common.DebugPrint(" aZero[5, 1] = %s" % str(aZero[5, 1]))
common.DebugPrint(" aZero[6, 1] = %s" % str(aZero[6, 1]))
aZero[5, 1] = 0
aZero[6, 1] = 0
self.assertTrue((aZero == 0).all())
def ComputeHarlocs(capture,
counterStep,
folderName,
fileNamePrefix,
fileNameExtension=".csv",
indexVideo=-1):
print( \
"Entered ComputeHarlocs(capture=%s, counterStep=%d, folderName=%s, " \
"indexVideo=%d)" % \
(str(capture), counterStep, folderName, indexVideo))
harlocsFolder = config.VIDEOS_FOLDER + "/" + folderName
t1 = float(cv2.getTickCount())
harlocs = []
if config.OCV_OLD_PY_BINDINGS:
numFrames = int(capture.get(cv2.cv.CV_CAP_PROP_FRAME_COUNT))
else:
numFrames = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))
common.DebugPrint("ComputeHarlocs(): numFrames = %d" % numFrames)
if not os.path.exists(harlocsFolder):
os.makedirs(harlocsFolder)
else:
#!!!!TODO: check that the loaded Harlocs are complete - same frame numbers as in the videos
# Folder with precomputed Harris features exists
folderContent = os.listdir(harlocsFolder)
sortedFolderContent = sorted(folderContent)
for fileName in sortedFolderContent:
pathFileName = harlocsFolder + "/" + fileName
"""
common.DebugPrint("ComputeHarlocs(): pathFileName = %s" % pathFileName);
common.DebugPrint("ComputeHarlocs(): fileName = %s" % fileName);
"""
if os.path.isfile(pathFileName) and \
fileName.startswith(fileNamePrefix) and \
pathFileName.endswith(fileNameExtension):
common.DebugPrint("ComputeHarlocs(): Loading %s" %
pathFileName)
harrisFeatures = multi_scale_harris.LoadMultiScaleHarrisFeatures(
pathFileName)
harlocs.append(harrisFeatures)
if config.endFrame[indexVideo] == -1:
assert (len(harlocs) + config.initFrame[indexVideo]) == numFrames
#!!!!TODO: if condition is NOT met, give a nicer error, or redo computations of Harlocs
else:
assert (len(harlocs) + config.initFrame[indexVideo]
) == config.endFrame[indexVideo] + 1
#!!!!TODO: if condition is NOT met, give a nicer error, or redo computations of Harlocs
return harlocs
if config.USE_MULTITHREADING == True:
global g
g.captureQ = None
# We need to reopen the capture device in each process, separately
g.captureR = None
# We need to reopen the capture device in each process, separately
#g.capture = capture;
g.harlocsFolder = harlocsFolder
g.fileNamePrefix = fileNamePrefix
g.fileNameExtension = fileNameExtension
g.indexVideo = indexVideo
if config.OCV_OLD_PY_BINDINGS:
frameCount = int(capture.get(cv2.cv.CV_CAP_PROP_FRAME_COUNT))
else:
frameCount = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))
#listParams = range(frameCount);
listParams = range(config.initFrame[indexVideo], frameCount,
counterStep)
common.DebugPrint("ComputeHarlocs(): frameCount = %d" % frameCount)
print("ComputeHarlocs(): Spawning a pool of %d workers" % \
config.numProcesses)
"""
# DEBUG purposes ONLY - since when we use Pool() and call function, if
# we have an error in the function the exception reported is very
# vague...
for i in listParams:
IterationStandaloneMSH(i);
#import time
#time.sleep(1000);
"""
"""
Start worker processes to use on multi-core processor (circumvent
also the GIL issue).
"""
pool = multiprocessing.Pool(processes=config.numProcesses)
print("ComputeHarlocs(): Spawned a pool of %d workers" % \
config.numProcesses)
#res = pool.map(IterationStandaloneMSH, listParams);
# See https://docs.python.org/2/library/multiprocessing.html#module-multiprocessing.pool
res = pool.map(func=IterationStandaloneMSH, iterable=listParams, \
chunksize=1)
print("Pool.map returns %s" % str(res))
"""
From https://medium.com/building-things-on-the-internet/40e9b2b36148
close the pool and wait for the work to finish
"""
pool.close()
pool.join()
#!!!!TODO: do more efficient - don't load the results from the CSV files
return ComputeHarlocs(capture, counterStep, folderName, \
fileNamePrefix, fileNameExtension, indexVideo)
#return [];
#indexHarloc = 0;
while True:
if config.OCV_OLD_PY_BINDINGS:
framePos = capture.get(cv2.cv.CV_CAP_PROP_POS_FRAMES)
else:
"""
From http://docs.opencv.org/modules/highgui/doc/reading_and_writing_images_and_video.html#videocapture-get:
<<CV_CAP_PROP_POS_FRAMES 0-based index of the frame to be decoded/captured next.>>
"""
framePos = capture.get(cv2.CAP_PROP_POS_FRAMES)
common.DebugPrint("ComputeHarlocs(): framePos = %d" % framePos)
counter = int(framePos)
#0
common.DebugPrint("ComputeHarlocs(): counter = %d" % counter)
ret, img = capture.read()
if common.MY_DEBUG_STDOUT:
common.DebugPrint("ComputeHarlocs(): img = %s" % str(img))
if False and config.SAVE_FRAMES:
fileName = config.IMAGES_FOLDER + "/img_%05d.png" % counter
if not os.path.exists(fileName):
#print "dir(img) = %s"% str(dir(img))
"""
imgCV = cv.fromarray(img)
cv2.imwrite(fileName, imgCV)
"""
cv2.imwrite(fileName, img)
#if ret == False: #MatchFrames.counterQ == 3:
if (ret == False) or ((counter > numFrames) or \
(config.endFrame[indexVideo] != -1 and \
counter > config.endFrame[indexVideo])):
break
if config.VIDEO_FRAME_RESIZE_SCALING_FACTOR != 1:
img = Matlab.imresize(img, \
scale=config.VIDEO_FRAME_RESIZE_SCALING_FACTOR)
im = img
pp = multi_scale_harris.multi_scale_harris(im, nos, disp=0)
# n=0:nos-1
#harlocs = pp
harlocs.append(pp)
multi_scale_harris.StoreMultiScaleHarrisFeatures( \
harlocsFolder + "/" + fileNamePrefix + "%05d%s" % \
(counter, fileNameExtension),
pp)
counter += counterStep
# If we try to seek to a frame out-of-bounds frame it gets to the last one
if config.OCV_OLD_PY_BINDINGS:
capture.set(cv2.cv.CV_CAP_PROP_POS_FRAMES, counter)
else:
capture.set(cv2.CAP_PROP_POS_FRAMES, counter)
#indexHarloc += 1;
t2 = float(cv2.getTickCount())
myTime = (t2 - t1) / cv2.getTickFrequency()
common.DebugPrint("ComputeHarlocs(): computing the multiscale harlocs " \
"took %.6f [sec]" % myTime)
#common.DebugPrint("ComputeHarlocs(): len(harlocs) = %s" % str(len(harlocs)));
if False:
for i, h in enumerate(harlocs):
#common.DebugPrint("ComputeHarlocs(): len(harlocs[%d]) = %d" % \
# (i, len(h)));
multi_scale_harris.StoreMultiScaleHarrisFeatures(
harlocsFolder + "/" + fileNamePrefix + "%05d.txt" % i, h)
#if False:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("ComputeHarlocs(): harlocs = %s" % str(harlocs))
return harlocs
def output_crossref_images(crossref, capture_q, capture_r):
if not os.path.exists(config.FRAME_PAIRS_MATCHES_FOLDER):
os.makedirs(config.FRAME_PAIRS_MATCHES_FOLDER)
rframe = None
gray_rframe = None
last_r_idx = None
t0 = float(cv2.getTickCount())
for q_idx, r_idx in crossref:
q_idx = int(q_idx)
r_idx = int(r_idx)
if common.MY_DEBUG_STDOUT:
t1 = float(cv2.getTickCount())
qframe = MyImageRead(capture_q, q_idx, grayscale=False)
gray_qframe = common.ConvertImgToGrayscale(qframe)
if r_idx != last_r_idx:
rframe = MyImageRead(capture_r, r_idx, grayscale=False)
gray_rframe = common.ConvertImgToGrayscale(rframe)
last_r_idx = r_idx
if common.MY_DEBUG_STDOUT:
t2 = float(cv2.getTickCount())
common.DebugPrint(
"ecc_homo_spacetime(): grabbing frames took %.6f [sec]" %
((t2 - t1) / cv2.getTickFrequency()))
# Default diff is a G-channel swap.
gdiff = np.zeros(qframe.shape)
gdiff[:, :, 0] = gray_qframe
gdiff[:, :, 1] = gray_rframe # G-Channel Swap
gdiff[:, :, 2] = gray_qframe
if config.VISUAL_DIFF_FRAMES:
# Compute difference between frames.
diff_mask = cv2.absdiff(qframe, rframe)
for i in range(diff_mask.shape[2]):
diff_mask[:, :, i] = cv2.GaussianBlur(src=diff_mask[:, :, i],
ksize=(7, 7),
sigmaX=10)
diff_mask_sum = diff_mask.sum(axis=2)
r_diff, c_diff = np.nonzero(
diff_mask_sum >= config.MEANINGFUL_DIFF_THRESHOLD *
diff_mask.shape[2])
meaningful_indices = zip(r_diff, c_diff)
# Create all black image so we can set the diff pixels to white.
diff_mask = np.zeros((diff_mask.shape[0], diff_mask.shape[1]),
dtype=np.uint8)
diff_mask[(r_diff, c_diff)] = 255
assert r_diff.size == c_diff.size
assert r_diff.size == len(meaningful_indices)
res = cv2.findContours(image=diff_mask,
mode=cv2.RETR_TREE,
method=cv2.CHAIN_APPROX_SIMPLE)
contours = res[1]
color_c = 255
meaningful_contours = 0
diff_mask = np.zeros((diff_mask.shape[0], diff_mask.shape[1]),
dtype=np.uint8)
for index_c, contour in enumerate(contours):
if len(contour) < 15:
continue
else:
cv2.drawContours(image=diff_mask,
contours=[contour],
contourIdx=0,
color=color_c,
thickness=-1)
meaningful_contours += 1
cv2.imwrite(
config.FRAME_PAIRS_MATCHES_FOLDER +
("%.6d_diff_mask" % q_idx) + config.imformat,
diff_mask.astype(int))
if config.SHOW_MASKED_DIFF:
qframe_diff = cv2.bitwise_and(qframe, qframe, mask=diff_mask)
rframe_diff = cv2.bitwise_and(rframe, rframe, mask=diff_mask)
# Save the masked diffs.
cv2.imwrite(
config.FRAME_PAIRS_MATCHES_FOLDER +
("%.6d_query_diff" % q_idx) + config.imformat,
qframe_diff.astype(int))
cv2.imwrite(
config.FRAME_PAIRS_MATCHES_FOLDER +
("%.6d_ref_diff" % q_idx) + config.imformat,
rframe_diff.astype(int))
if config.SAVE_FRAMES:
# We use q_idx because ref video is aligned against the query video.
# These frames represent the aligned output.
cv2.imwrite(
config.FRAME_PAIRS_MATCHES_FOLDER + ("%.6d_query" % q_idx) +
config.imformat, qframe.astype(int))
cv2.imwrite(
config.FRAME_PAIRS_MATCHES_FOLDER + ("%.6d_ref" % q_idx) +
config.imformat, rframe.astype(int))
cv2.imwrite(
config.FRAME_PAIRS_MATCHES_FOLDER + ("%.6d_gchan_diff" % q_idx) +
config.imformat, gdiff.astype(int))
print("Saving crossref frames took %.6f [sec]" %
((float(cv2.getTickCount()) - t0) / cv2.getTickFrequency()))
def temporal_alignment(counter_q, frame_q, capture_r, num_frames_r,
num_features_matched, f_output):
global p1, p2, kp_pairs, status, H, nonp1
if config.USE_EXHAUSTIVE_SEARCH:
max_features_matched = -2000000000
pos_max_features_matched = -1
while True:
if config.OCV_OLD_PY_BINDINGS:
frameR = capture_r.get(cv2.cv.CV_CAP_PROP_POS_FRAMES)
else:
frameR = capture_r.get(cv2.CAP_PROP_POS_FRAMES)
common.DebugPrint("Alex: frameR = %d" % frameR)
counter_r = int(frameR)
ret2, img_r = capture_r.read()
if (ret2 == False) or (counter_r > num_frames_r):
break
if config.SAVE_FRAMES:
file_name = config.IMAGES_FOLDER + "/imgR_%05d.png" % counter_r
if not os.path.exists(file_name):
cv2.imwrite(file_name, img_r)
res = compute_features_and_match_2(img_r, counter_r)
cost_used = 1
if cost_used == 0:
pass
else:
# myTemporalAlignmentCost is the sum of distances of best-pairs
# (closest neighbors) of matched features
res = (res[0], -myTemporalAlignmentCost)
num_features_matched[counter_q][counter_r] = res[1]
compute_best_fast = False
if max_features_matched < res[1]:
max_features_matched = res[1]
pos_max_features_matched = counter_r
if config.SAVE_FRAMES:
if not compute_best_fast:
# TODO: don't do it even for best candidates -
# only for the best ONE - this implies redoing probably
# some computation from temporal_alignment() for the
# best frame pair
# We call rest() in order to compute vis and visOrig.
rest("Image Match")
vis_best = vis.copy()
vis_orig_best = vis_orig.copy()
else:
p1_best = p1
p2_best = p2
kp_pairs_best = kp_pairs
status_best = status
h_best = H
nonp1_best = nonp1
common.DebugPrint("Alex: counter_r = %d" % counter_r)
counter_r += config.counterRStep
"""
If we try to seek to a frame out-of-bounds frame it gets to
the last one.
"""
if config.OCV_OLD_PY_BINDINGS:
capture_r.set(cv2.cv.CV_CAP_PROP_POS_FRAMES, counter_r)
else:
capture_r.set(cv2.CAP_PROP_POS_FRAMES, counter_r)
common.DebugPrint(
"Alex: Time = %s" %
common.GetCurrentDateTimeStringWithMilliseconds())
my_text = " Frame #%d of video A: frame #%d of video B, " \
"with %d features matched (time = %s)" % \
(counter_q, pos_max_features_matched, max_features_matched,
common.GetCurrentDateTimeStringWithMilliseconds())
print(my_text, file=f_output)
f_output.flush()
counter_r_best = pos_max_features_matched
else:
# We empty the memoization cache before SimAnneal.main()
SimAnneal.Acache = {}
res = SimAnneal.main()
res2 = (res[0], -res[1])
common.DebugPrint(
"Best solution for frame counter_q=%d is %s. Time = %s" %
(counter_q, str(res2),
common.GetCurrentDateTimeStringWithMilliseconds()))
# TODO: check if OK:
counter_r_best = res[0]
if compute_best_fast:
# TODO: check that these assignments really refer the objects defined
# above (when updating) and that there are no escapes of the
# values/objects that result in side-effects updating the respective
# objects and messing everything up - better said we look if
# redefinitons of the rhs are done inside (some of their subelements)
# or totally (reassign a completely NEW object).
p1 = p1_best
p2 = p2_best
kp_pairs = kp_pairs_best
status = status_best
H = h_best
nonp1 = nonp1_best
rest("Image Match")
vis_best = vis.copy()
vis_orig_best = vis_orig.copy()
if config.SAVE_FRAMES:
"""
We display frames imgQ and img_r with features (from temporal) and
clusters on them.
"""
cv2.imwrite(
config.FRAME_PAIRS_MATCHES_FOLDER + "/img_proc_%05d_%05d.png" %
(counter_q, counter_r_best), vis_best)
# We display also the orig frames imgQ and img_r
cv2.imwrite(
config.FRAME_PAIRS_MATCHES_FOLDER +
"/img_proc_%05d_%05d_orig.png" % (counter_q, counter_r_best),
vis_orig_best)
"""
We return the len of the status returned by cv2.findHomography() - this is
THE indicator of the match of a pair of frames.
NOTE: we don't use the result :)))
!!!!TODO: think to take it out
"""
return res[1] # res
def feature_match_and_homography():
# TODO: memoize (but it's gonna be huge) the pair results of matching -
# should reduce by 2 the runtime
global p1, p2, kp_pairs, status, H
print("Entered feature_match_and_homography()...")
# raw_matches still has the same len as desc1[counter_q] AND trainDescriptors
t1 = float(cv2.getTickCount())
"""
opencv2refman.pdf, Section 7.4., page 429:
knnMatch
"Finds the k best matches for each descriptor from a query set."
"k - Count of best matches found per each query descriptor or less if a
query descriptor has less than k possible matches in total."
"""
raw_matches = matcher.knnMatch(desc1[counter_q],
trainDescriptors=desc2[counter_r],
k=2)
p1, p2, kp_pairs = filter_keypoints_matches(kp1[counter_q], kp2[counter_r],
raw_matches)
t2 = float(cv2.getTickCount())
my_time = (t2 - t1) / cv2.getTickFrequency()
common.DebugPrint("feature_match_and_homography(): knnMatch() and "
"filter_keypoints_matches() took %.6f [sec]" % my_time)
"""
# We cluster & display the non-matched keypoints
nonp1 = cluster_unmatched_keypoints(nonp1)
"""
"""
raw_matches is a list of corresponding?? pairs of DMatch for the keypoints
obtained with knnMatch() .
p1, p2 are only the matched keypoints from the 2 frames
kp_pairs is pair of corresponding keypoints, obtained with the list zip
operation:
cardinality ~=???? the cardinality of kp1[counter_q] and kp2[counter_r]
"""
if len(p1) >= 4:
# p1 and p2 are the matched keypoints
"""
From Section 6.1, page 391, opencv2refman.pdf:
(also http://docs.opencv.org/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.html?highlight=findhomography#findhomography):
"Finds a perspective transformation between two planes."
"However, if not all of the point pairs (srcPoints
i ,:math:dstPoints_i ) fit the rigid perspective transformation
(that is, there are some outliers), this initial estimate will be
poor."
"In this case, you can use one of the two robust methods.
Both methods, RANSAC and LMeDS , try many different random subsets
of the corresponding point pairs (of four pairs each), estimate
the homography matrix using this subset and a simple least-square
algorithm, and then compute the quality/goodness of the computed
homography (which is the number of inliers for RANSAC or the median
re-projection error for LMeDs). The best subset is then used to
produce the initial estimate of the homography matrix and the
mask of inliers/outliers."
"""
t1 = float(cv2.getTickCount())
H, status = cv2.findHomography(srcPoints=p1,
dstPoints=p2,
method=cv2.RANSAC,
ransacReprojThreshold=5.0)
t2 = float(cv2.getTickCount())
my_time = (t2 - t1) / cv2.getTickFrequency()
common.DebugPrint(
"feature_match_and_homography(): cv2.findHomography() "
"took %.6f [sec]" % my_time)
if status is None:
status = []
if H == None:
print("!!!!!!!!found H None - len(p1) = %d" % len(p1))
H = []
return -1, len(p1)
print("%d / %d inliers/matched (len(p1) = %d)" %
(np.sum(status), len(status), len(p1)))
# Note: H is the homography matrix, a 3x3 matrix.
common.DebugPrint(
"H, the homography matrix, from cv2.findHomography = %s" % str(H))
common.DebugPrint(" len(H) = %d" % len(H))
# Note that len(status) == len(p1)
res = (np.sum(status), len(status))
else:
H, status = None, None
common.DebugPrint(
"%d matches found, not enough for homography estimation" % len(p1))
res = (-1, len(p1))
# The result is related to the homography transformation: status sum and len
return res
def hierarchical_clustering_with_cv2_unfinished(z, n):
# We choose an ~optimal number of clusters
min_validity = 1000000
min_validity_k = -1
for k in range(2, 10 + 1):
a = [None] * k
"""
Inspired a bit from
https://www.google-melange.com/gsoc/project/google/gsoc2013/abidrahman2/43002,
\source\py_tutorials\py_ml\py_kmeans\py_kmeans_opencv\py_kmeans_opencv.rst
"""
# Define criteria and apply kmeans()
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 10, 1.0)
t1 = float(cv2.getTickCount())
ret, label, center = cv2.kmeans(z, k, criteria, 10,
cv2.KMEANS_RANDOM_CENTERS)
t2 = float(cv2.getTickCount())
my_time = (t2 - t1) / cv2.getTickFrequency()
common.DebugPrint(
"HierarchicalClusteringWithCV2_UNFINISHED(): "
"cv2.kmeans() took %.5f [sec]" % my_time)
common.DebugPrint("ret = %s" % str(ret))
common.DebugPrint("label = %s" % str(label))
common.DebugPrint("center = %s" % str(center))
# Now separate the data, Note the flatten()
for i in range(k):
a[i] = z[label.ravel() == i]
common.DebugPrint("A[0] = %s" % str(a[0]))
common.DebugPrint("A[0][:, 0] = %s" % str(a[0][:, 0]))
common.DebugPrint("A[0][:, 1] = %s" % str(a[0][:, 1]))
"""
Following Section 3.2 from http://www.csse.monash.edu.au/~roset/papers/cal99.pdf :
See if you have the time, for further ideas:
https://stackoverflow.com/questions/15376075/cluster-analysis-in-r-determine-the-optimal-number-of-clusters
"""
intra = 0
for i in range(k):
# Gives exception: "TypeError: only length-1 arrays can be converted to Python scalars"
for x in range(len(a[i])):
intra += np.square(a[i][x, 0] - center[i, 0]) + \
np.square(a[i][x, 1] - center[i, 1])
intra /= n
dist_min = 1000000
for i in range(k):
for j in range(i + 1, k):
dist = np.square(center[i, 0] - center[j, 0]) + \
np.square(center[i, 1] - center[j, 1])
"""
dist = sqr(center[i, 0] - center[j, 0]) + \
sqr(center[i, 1] - center[j, 1])
"""
common.DebugPrint("dist = %s" % str(dist))
if dist < dist_min:
dist_min = dist
inter = dist_min
"""
We want to minimize intra (clusters be dense) and
maximize inter (clusters be distant from one another).
"""
validity = intra / inter
if min_validity > validity:
min_validity = validity
min_validity_k = k
if config.USE_GUI:
# We clear the figure and the axes
plt.clf()
plt.cla()
# Plot the data
for i in range(k):
"""
Note: A[0][:,0] (i.e., [:,0] is a numpy-specific
"split"-operator, not working for standard Python lists.
"""
plt.scatter(a[i][:, 0], a[i][:, 1], c=colors[i])
plt.scatter(center[:, 0], center[:, 1], s=80, c="b", marker="s")
plt.xlabel(
"Height. Also: k=%d, intra=%.1f, inter=%.1f, validity = %.4f" %
(k, intra, inter, validity))
plt.ylabel("Weight")
plt.show()
"""
TODO!!!! Implement section 4 from http://www.csse.monash.edu.au/~roset/papers/cal99.pdf:
See "when we require the number of
clusters to be increased, we split the cluster
having maximum variance, so the k-means
procedure is given good starting cluster centres."
"""
# !!!!TODO: .... DO THE IMPLEMENTATION, WHITE BOY
common.DebugPrint("IMPORTANT: min_validity_k = %d" % min_validity_k)
def hierarchical_clustering(z, n):
n = len(z)
common.DebugPrint("hierarchical_clustering(): n = %d" % n)
# Note: Z is not standard list, but a numpy array
if len(z) < 10: # or Z == []:
common.DebugPrint("hierarchical_clustering(): Bailing out of "
"hierarchical clustering since too few elements "
"provided (and I guess we could have issues)")
return []
# Vector of (N choose 2) pairwise Euclidian distances
d_sch = sch.distance.pdist(z)
d_max = d_sch.max()
# This parameter is CRUCIAL for the optimal number of clusters generated
# This parameter works better for the videos from Lucian
threshold = 0.05 * d_max
"""
I did not find much information on the linkage matrix (linkage_matrix), but
from my understanding it is the direct result of the hierarchical
clustering, which is performed by recursively splitting clusters,
forming a dendrogram forest of trees (see if you have time
https://stackoverflow.com/questions/5461357/hierarchical-k-means-in-opencv-without-knowledge-of-k
"a forest of hierarchical clustering trees").
The linkage matrix is stores on each row data for a clustered point:
- the last element in the row is the leaf in the dendrogram tree
forest the point belongs to. The leaf does not really tell you
to which final cluster the point belongs to - (IMPORTANT) for
this, we have the function sch.fcluster().
See if you have the time (for some better understanding):
https://stackoverflow.com/questions/11917779/how-to-plot-and-annotate-hierarchical-clustering-dendrograms-in-scipy-matplotlib
See doc:
http://docs.scipy.org/doc/scipy/reference/generated/scipy.cluster.hierarchy.linkage.html#scipy.cluster.hierarchy.linkage
See if you have the time:
https://stackoverflow.com/questions/16883412/how-do-i-get-the-subtrees-of-dendrogram-made-by-scipy-cluster-hierarchy
"""
linkage_matrix = sch.linkage(d_sch, "single")
common.DebugPrint("linkage_matrix = %s" % str(linkage_matrix))
# Inspired from https://stackoverflow.com/questions/7664826/how-to-get-flat-clustering-corresponding-to-color-clusters-in-the-dendrogram-cre
index_cluster = sch.fcluster(linkage_matrix, threshold, "distance")
common.DebugPrint("index_cluster = %s" % str(index_cluster))
c_max = -1
# We "truncate" later the ending zeros from num_elems
num_elems = [0] * (n + 1)
# IMPORTANT: It appears the ids of the clusters start from 1, not 0
for e in index_cluster:
# print "e = %s" % str(e)
num_elems[e] += 1
if c_max < e:
c_max = e
# c_max is the MAXIMUM optimal number of clusters after the hierarchical
# clustering is performed.
common.DebugPrint("c_max (the MAX id of final clusters) = %d" % c_max)
num_elems = num_elems[0 : c_max + 1]
common.DebugPrint("num_elems = %s" % str(num_elems))
"""
# We can also use:
num_elems.__delslice__(c_max + 1, len(num_elems))
but it's sort of deprecated
- see http://docs.python.org/release/2.5.2/ref/sequence-methods.html
"""
num_clusters = 0
for e in num_elems:
if e != 0:
num_clusters += 1
common.DebugPrint("num_clusters (the optimal num of clusters) = %d" %
num_clusters)
assert num_clusters == c_max
num_clusters_above_threshold = 0
for i in range(c_max + 1):
if num_elems[i] >= \
config.THRESHOLD_NUM_NONMATCHED_ELEMENTS_IN_CLUSTER:
common.DebugPrint("num_elems[%d] = %d" % (i, num_elems[i]))
num_clusters_above_threshold += 1
common.DebugPrint("num_clusters_above_threshold = %d" %
num_clusters_above_threshold)
# TODO: Move this to config?
return_only_biggest_cluster = False
if return_only_biggest_cluster:
# TODO: find biggest cluster - sort them after num_elems, etc
res = []
for i in range(n):
# We start numbering the clusters from 1.
if index_cluster[i] == num_clusters:
res.append(z[i])
else:
res = {}
for i in range(n):
if num_elems[index_cluster[i]] >= \
config.THRESHOLD_NUM_NONMATCHED_ELEMENTS_IN_CLUSTER:
if index_cluster[i] not in res:
res[index_cluster[i]] = []
res[index_cluster[i]].append(z[i])
if config.USE_GUI and config.DISPLAY_PYTHON_CLUSTERING:
# We clear the figure and the axes
plt.clf()
plt.cla()
# Plot the data
for i in range(n): # index_cluster:
# print "Z[i, 0] = %.2f, Z[i, 1] = %.2f" % (Z[i, 0], Z[i, 1])
try:
col_cluster = colors[index_cluster[i]]
except: # IndexError: list index out of range
col_cluster = 2
plt.scatter(z[i, 0], z[i, 1], c=col_cluster)
plt.xlabel(
"Height. (num_clusters = %d, num_clusters_above_threshold = %d)" %
(num_clusters, num_clusters_above_threshold))
plt.ylabel("Weight")
# From http://matplotlib.org/api/pyplot_api.html#matplotlib.pyplot.axis
v = plt.axis()
# We invert the y to have 0 up and x axis to have 0
v = (0, v[1], v[3], 0)
plt.axis(v)
plt.show()
return res
def QuadTreeDecision():
"""
global r_path, q_path
common.DebugPrint("QuadTreeDecision(): r_path = %s" % r_path);
common.DebugPrint("QuadTreeDecision(): q_path = %s" % q_path);
"""
#global harlocsQ, harlocsR
common.DebugPrint("Entered QuadTreeDecision().")
totalT1 = float(cv2.getTickCount())
r_quadsTree = None
"""
Matlab code:
www=whos('tree');
if size(www,1)>0
kdtree_delete(tree);
clear tree;
end
"""
#clear Votes_space H;
Votes_space = None
H = None
#['arr_0']
if r_quadsTree != None:
# TODO: clear tree
pass
if method == 1:
#%% search among all reference quads
common.DebugPrint(
"\nQuadTreeDecision(): Search among all reference quads...(Tree method)"
)
try:
crossref = np.load("crossref.npz")['arr_0']
common.DebugPrint(
"\nQuadTreeDecision(): Found already precomputed crossref.npz - returning it)"
)
return crossref
except:
common.DebugPrintErrorTrace()
#BOV_flag=0;
BOV_flag = 0
foundFiles = False
try:
Votes_space = np.load("Votes_space.npz")['arr_0']
H = np.load("H.npz")['arr_0']
foundFiles = True
except:
common.DebugPrintErrorTrace()
if foundFiles == False:
"""
Alex: scale s is 1 for original frame resolution and the higher
we go we have lower image resolutions (we go higher in the
Guassian pyramid I think).
"""
#for s=1:nos
for s in range(1, nos + 1):
common.DebugPrint("QuadTreeDecision(): Scale %d" % s)
#md_threshold=round(s*100+100^(log(s)));
md_threshold = round(s * 100 + pow(100, math.log(s)))
#[tree,all_id,all_cen,all_max,all_ori,n_d,all_quads]=multiscale_quad_tree(r_path, md_threshold,s);
#tree, all_id, all_cen, all_max, all_ori, n_d, all_quads = multiscale_quad_tree.multiscale_quad_tree(r_path, md_threshold, s)
r_quadsTree, all_id, all_cen, all_max, all_ori, n_d, all_quads = \
multiscale_quad_tree.multiscale_quad_tree(harlocsR, \
md_threshold, s)
if config.PREPROCESS_REFERENCE_VIDEO_ONLY == True:
continue
if r_quadsTree == None:
continue
common.DebugPrint("QuadTreeDecision(): md_threshold = %s" %
str(md_threshold))
#[Votes_space(:,:,s),H(:,:,s)]=multiscale_quad_retrieval(tree, r_path, q_path, md_threshold, st_threshold, all_ori, all_id, all_max, all_cen,nos, s, cropflag, sequence);
# Votes_space(:,:,s),H(:,:s) =multiscale_quad_retrieval(tree, r_path, q_path, md_threshold, st_threshold, all_ori, all_id, all_max, all_cen, nos, s, cropflag, sequence)
#Votes_space[:, :, s - 1], H[:, :,s - 1] = multiscale_quad_retrieval.multiscale_quad_retrieval(r_quadsTree, harlocsR, harlocsQ, md_threshold, st_threshold, all_ori, all_id, all_max, all_cen, nos, s, cropflag, sequence)
Votes_space_res, H_res = multiscale_quad_retrieval.multiscale_quad_retrieval(r_quadsTree, \
harlocsR, harlocsQ, md_threshold, \
st_threshold, all_ori, all_id, \
all_max, all_cen, nos, s, cropflag, \
sequence)
if Votes_space == None:
Votes_space = np.zeros((Votes_space_res.shape[0],
Votes_space_res.shape[1], nos))
"""
Inspired from https://stackoverflow.com/questions/17559140/matlab-twice-as-fast-as-numpy
BUT doesn't help in this case:
Votes_space = np.asfortranarray(np.zeros( (Votes_space_res.shape[0], Votes_space_res.shape[1], nos) ));
"""
if H == None:
H = np.zeros((H_res.shape[0], H_res.shape[1], nos),
dtype=np.int8)
"""
Inspired from https://stackoverflow.com/questions/17559140/matlab-twice-as-fast-as-numpy
BUT doesn't help in this case:
H = np.asfortranarray(np.zeros( (H_res.shape[0], H_res.shape[1], nos) ));
"""
Votes_space[:, :, s - 1] = Votes_space_res
H[:, :, s - 1] = H_res
if common.MY_DEBUG_STDOUT:
common.DebugPrint("QuadTreeDecision(): For scale %d: " \
"Votes_space_res = %s,\n H_res = %s" % \
(s, str(Votes_space_res), str(H_res)))
common.DebugPrint("QuadTreeDecision(): For scale %d: " \
"Votes_space_res.shape = %s,\n H_res.shape = %s" % \
(s, str(Votes_space_res.shape), str(H_res.shape)))
#quit();
#kdtree_delete(tree); # TODO: think if want to delete kdtree
if config.KDTREE_IMPLEMENTATION == 1:
r_quadsTree.release()
if config.PREPROCESS_REFERENCE_VIDEO_ONLY == True:
common.DebugPrint("QuadTreeDecision(): Exiting program " \
"since we finished preprocessing the reference video")
common.DebugPrint("QuadTreeDecision(): time before exit = %s" % \
common.GetCurrentDateTimeStringWithMilliseconds())
return None
#quit();
if common.MY_DEBUG_STDOUT:
common.DebugPrint("QuadTreeDecision(): Before multiscale_synchro_decision(): " \
"Votes_space = %s,\n H = %s" % (str(Votes_space), str(H)))
try:
# See http://docs.scipy.org/doc/numpy/reference/generated/numpy.savez.html
np.savez_compressed("Votes_space", Votes_space)
np.savez_compressed("H", H)
except:
common.DebugPrintErrorTrace()
#q_path = [None] * len(harlocsQ);
numFramesQ = len(harlocsQ)
#r_path = [None] * len(harlocsR);
numFramesR = len(harlocsR)
if config.temporalDecisionType == 1:
# causal solution - "local"
#cross=multiscale_synchro_decision(Votes_space, H, q_path, r_path, BOV_flag, cropflag, const_type);
crossref = multiscale_synchro_decision.causal( \
Votes_space, H, numFramesQ, numFramesR, BOV_flag, cropflag, \
const_type)
# str=['save ' q_path 'cross_baseline cross'];
# eval(str)
elif config.temporalDecisionType == 0:
# decision (non-causal solution)
#[y,x,D,Tback,cross] = dp3(Votes_space, r_path, q_path, BOV_flag);
y, x, D, Tback, crossref = multiscale_synchro_decision.dp3( \
Votes_space, numFramesR, numFramesQ, BOV_flag)
# str=['save ' q_path 'cross_baseline_dp cross'];
# eval(str)
else:
"""
!!!!TODO: implement if useful VD (or BoW)
NOTE: see config.py for Evangelidis' comments from email of Apr 14, 2014:
Basically he argues that:
- the VD method is similar in quality with the full-search VS
- BoW is not great.
"""
assert False
# not implemented
crossref[:, 1] += config.initFrame[1]
#myText = "crossref = \n%s" % crossref;
myText = ""
for r in range(crossref.shape[0]):
myText += " %d %d\n" % (crossref[r][0], crossref[r][1])
fOutput = open("crossref.txt", "wt")
fOutput.write(myText)
fOutput.close()
try:
# See http://docs.scipy.org/doc/numpy/reference/generated/numpy.savez.html
np.savez_compressed("crossref", crossref)
except:
common.DebugPrintErrorTrace()
totalT2 = float(cv2.getTickCount())
myTime = (totalT2 - totalT1) / cv2.getTickFrequency()
print("QuadTreeDecision() took %.6f [sec]" % (myTime))
return crossref
