def TemporalAlignment(captureQ, captureR): global harlocsQ, harlocsR # if compute_harris == 1:... totalT1 = float(cv2.getTickCount()) #if config.PREPROCESS_REFERENCE_VIDEO_ONLY == True: # This will give error in multiscale_quad_retrieval since we need to load the saved Harris features anyhow if True: # We compute and Store in files the multi-scale Harris features of the reference video """ harlocsR = ComputeHarlocs(captureR, config.counterRStep, \ folderName="/harlocs_ref", fileNamePrefix="harloc"); """ harlocsR = ComputeHarlocs(captureR, config.counterRStep, \ folderName=config.HARRIS_REFERENCE_FOLDER_NAME, \ fileNamePrefix=config.HARRIS_FILENAME_PREFIX, \ fileNameExtension=config.HARRIS_FILENAME_EXTENSION, indexVideo=1) if common.MY_DEBUG_STDOUT: common.DebugPrint("TemporalAlignment(): len(harlocsR) = %s" % str(len(harlocsR))) sumNbytes = 0 for hr in harlocsR: sumNbytes += hr.nbytes common.DebugPrint("TemporalAlignment(): harlocsR.nbytes = %s" % str(sumNbytes)) #common.DebugPrint("TemporalAlignment(): harlocsR.nbytes = %s" % str(harlocsR.nbytes)); if config.PREPROCESS_REFERENCE_VIDEO_ONLY == False: # We compute and Store in files the multi-scale Harris features of the query video """ Note: if the "harloc" files exist, load directly the features from the files without computing them again. """ harlocsQ = ComputeHarlocs(captureQ, config.counterQStep, \ folderName=config.HARRIS_QUERY_FOLDER_NAME, \ fileNamePrefix=config.HARRIS_FILENAME_PREFIX, \ fileNameExtension=config.HARRIS_FILENAME_EXTENSION, indexVideo=0) if common.MY_DEBUG_STDOUT: common.DebugPrint("TemporalAlignment(): len(harlocsQ) = %s" % \ str(len(harlocsQ))) sumNbytes = 0 for hq in harlocsQ: sumNbytes += hq.nbytes common.DebugPrint("TemporalAlignment(): harlocsQ.nbytes = %s" % str(sumNbytes)) #res = QuadTreeDecision(captureQ, captureR); res = QuadTreeDecision() else: res = None totalT2 = float(cv2.getTickCount()) myTime = (totalT2 - totalT1) / cv2.getTickFrequency() print("TemporalAlignment() took %.6f [sec]" % (myTime)) return res
def process_input_frames(capture_q, capture_r, f_output): # Allocate num_features_matched num_features_matched = [None] * num_frames_q for i in range(num_frames_q): num_features_matched[i] = [-2000000000] * num_frames_r while True: if config.OCV_OLD_PY_BINDINGS: frame_q = capture_q.get(cv2.cv.CV_CAP_PROP_POS_FRAMES) else: frame_q = capture_q.get(cv2.CAP_PROP_POS_FRAMES) common.DebugPrint("Alex: frame_q = %d" % frame_q) counter_q = int(frame_q) common.DebugPrint("Alex: counter_q = %d" % counter_q) ret1, imgQ = capture_q.read() if False and config.SAVE_FRAMES: file_name = config.IMAGES_FOLDER + "/imgQ_%05d.png" % counter_q if not os.path.exists(file_name): cv2.imwrite(file_name, imgQ) if (ret1 == False) or (counter_q > num_frames_q): break # TODO: counter_q already visible in module MatchFrames compute_features_1(imgQ, counter_q) # We set the video stream capture_r at the beginning if config.OCV_OLD_PY_BINDINGS: capture_r.set(cv2.cv.CV_CAP_PROP_POS_FRAMES, 0) else: capture_r.set(cv2.CAP_PROP_POS_FRAMES, 0) # Start time profiling for the inner loop t1 = float(cv2.getTickCount()) # TODO: counter_q already visible in module MatchFrames temporal_alignment(counter_q, frame_q, capture_r, num_frames_r, num_features_matched, f_output) # Measuring how much it takes the inner loop t2 = float(cv2.getTickCount()) my_time = (t2 - t1) / cv2.getTickFrequency() common.DebugPrint( "Avg time it takes to complete a match (and to perform " "INITIAL Feat-Extract) = %.6f [sec]" % (my_time / (num_frames_r / config.counterRStep))) counter_q += config.counterQStep # If we try to seek to a frame out-of-bounds frame it gets to the # last one if config.OCV_OLD_PY_BINDINGS: capture_q.set(cv2.cv.CV_CAP_PROP_POS_FRAMES, counter_q) else: capture_q.set(cv2.CAP_PROP_POS_FRAMES, counter_q) common.DebugPrint("num_features_matched = %s" % str(num_features_matched))
def filter_keypoints_matches(kp1, kp2, matches, ratio=0.95): global nonp1, nonp2 global myTemporalAlignmentCost myTemporalAlignmentCost = 0 mkp1, mkp2 = [], [] nonp1, nonp2 = [], [] index = 0 for m in matches: len_m = len(m) if len_m == 2: if m[0].distance < m[1].distance * ratio: """ Inspired from http://www.cs.ubc.ca/~lowe/papers/ijcv04.pdf#page=20: "A more effective measure is obtained by comparing the distance of the closest neighbor to that of the second-closest neighbor." Note: m[0] = closest neighbor match m[1] = 2nd-closest neighbor match """ # These keypoints are considered matched - # we choose the 1st closest match mkp1.append(kp1[m[0].queryIdx]) mkp2.append(kp2[m[0].trainIdx]) myTemporalAlignmentCost += m[0].distance else: # These keypoints are considered NOT matched nonp1.append(kp1[m[0].queryIdx].pt) # TODO: I think that nonp2.append below is NOT relevant else: assert len_m <= 2 # These keypoints are considered NOT matched if len_m == 1: nonp1.append(kp1[m[0].queryIdx].pt) else: nonp1.append(kp1[index].pt) index += 1 """ nonp1 is a list of pairs of x, y coordinates for the features/keypoints that have not been matched from frame of video A. """ common.DebugPrint("len(nonp1) = %d" % len(nonp1)) common.DebugPrint("len(nonp2) = %d" % len(nonp2)) p1 = np.float32([kp.pt for kp in mkp1]) p2 = np.float32([kp.pt for kp in mkp2]) kp_pairs = zip(mkp1, mkp2) return p1, p2, kp_pairs
def onmouse(event, x, y, flags, param): crt_vis = vis if flags & cv2.EVENT_FLAG_LBUTTON: crt_vis = vis0.copy() r = 3 m = (common_cv.anorm(p1_local - (x, y)) < r) | \ (common_cv.anorm(p2_local - (x, y)) < r) idxs = np.where(m)[0] kp1s, kp2s = [], [] assert len(p1_local) == len(p2_local) for i in idxs: (x1, y1), (x2, y2) = p1_local[i], p2_local[i] try: col = (red, green)[status_local[i]] except: common.DebugPrintErrorTrace() common.DebugPrint("onmouse() exception: i = %d, " "len(statusLocal) = %d" % (i, len(status_local))) col = red cv2.line(crt_vis, (x1, y1), (x2, y2), col) try: kp1, kp2 = kp_pairs[i] kp1s.append(kp1) kp2s.append(kp2) except: common.DebugPrintErrorTrace() common.DebugPrint( "onmouse() exception2: i = %d, " "len(kp_pairs)=%d, len(idxs)=%d, idxs=%s" % (i, len(kp_pairs), len(idxs), str(idxs))) crt_vis = cv2.drawKeypoints(crt_vis, kp1s, flags=4, color=kp_color) crt_vis[:, w_q:] = cv2.drawKeypoints(crt_vis[:, w_q:], kp2s, flags=4, color=kp_color) cv2.imshow(win, crt_vis)
def compute_features_and_match_2(a_img_r, a_counter_r): global kp2, desc2 global counter_r global img_r img_r = a_img_r counter_r = a_counter_r if counter_r is None: counter_r = 0 common.DebugPrint( "compute_features_and_match_2(): counter_r = %d (counter_q = %d)" % (counter_r, counter_q)) # If we did NOT memoize kp2[counter_r], desc2[counter_r] we compute # them once if kp2[counter_r] is None: """ From http://docs.opencv.org/trunk/modules/nonfree/doc/feature_detection.html "Python API provides three functions. First one finds keypoints only. Second function computes the descriptors based on the keypoints we provide. Third function detects the keypoints and computes their descriptors If you want both keypoints and descriptors, directly use third function as kp, des = cv2.SIFT.detectAndCompute(image, None)" """ t1 = float(cv2.getTickCount()) kp2[counter_r], desc2[counter_r] = detector.detectAndCompute( a_img_r, None) t2 = float(cv2.getTickCount()) my_time = (t2 - t1) / cv2.getTickFrequency() common.DebugPrint("compute_features_and_match_2(): " "detector.detectAndCompute() took %.6f [sec]" % my_time) common.DebugPrint("a_img_r - %d features" % len(kp2[counter_r])) res = feature_match_and_homography() """ The result is related to the homography transformation returned by feature_match_and_homography(): status sum and len """ return res
def IterationStandaloneMSH(index): #capture = g.capture; harlocsFolder = g.harlocsFolder fileNamePrefix = g.fileNamePrefix fileNameExtension = g.fileNameExtension common.DebugPrint("Entered IterationStandaloneMSH(index=%d)" % index) #img = MyImageReadMSH(capture, index); img = MyImageReadMSH(index) im = img pp = multi_scale_harris.multi_scale_harris(im, nos, disp=0) # n=0:nos-1 if False: #harlocs = pp harlocs.append(pp) multi_scale_harris.StoreMultiScaleHarrisFeatures( \ harlocsFolder + "/" + fileNamePrefix + "%05d%s" % \ (index, fileNameExtension), pp) if False: counter += counterStep # If we try to seek to a frame out-of-bounds frame it gets to the last one if config.OCV_OLD_PY_BINDINGS: capture.set(cv2.cv.CV_CAP_PROP_POS_FRAMES, counter) else: capture.set(cv2.CAP_PROP_POS_FRAMES, counter) return 1
def rest(win): global nonp1 global status # Here we perform spatial alignment and clustering and display the results. t1 = float(cv2.getTickCount()) # TODO: filter out p1 outside the alignment of the pair of frames (myVis, inputFrame, refFrame, status2) = spatial_alignment(win, img_q, img_r, kp_pairs, status, H) t2 = float(cv2.getTickCount()) my_time = (t2 - t1) / cv2.getTickFrequency() common.DebugPrint( "feature_match_and_homography(): spatial_alignment() took %.6f [sec]" % my_time) # TODO: filter out nonp1 outside the alignment of the pair of frames - # should do it on p1 inside spatial_alignment() # We cluster the non-matched keypoints nonp1 = cluster_unmatched_keypoints(nonp1) my_vis2 = annotate_vis(win, myVis, inputFrame, refFrame, status) # TODO: remove global definition of vis global vis vis = my_vis2 if config.USE_GUI: ch = cv2.waitKey() print("ch = %s" % str(ch)) if ch == 27: # Escape key quit() cv2.destroyAllWindows()
def write_video_capture(video_file_name, folder_name): # OpenCV can read only AVIs - not 3GP, nor FLVs with MPEG compression # From http://answers.opencv.org/question/6/how-to-readwrite-video-with-opencv-in-python/ video_writer = None folder_content = os.listdir(folder_name) sorted_folder_content = sorted(folder_content) for fileName in sorted_folder_content: path_file_name = folder_name + "/" + fileName if os.path.isfile(path_file_name) and \ fileName.lower().endswith("_good.png"): common.DebugPrint("ComputeHarlocs(): Loading %s" % path_file_name) img = cv2.imread(path_file_name) assert img is not None if video_writer: common.DebugPrint("img.shape = %s" % str(img.shape)) # From http://docs.opencv.org/trunk/doc/py_tutorials/py_gui/py_video_display/py_video_display.html#saving-a-video # See also http://opencv-python-tutroals.readthedocs.org/en/latest/py_tutorials/py_gui/py_video_display/py_video_display.html # WRITES 0 BYTES IN THE VIDEO: vid_fourcc = cv2.VideoWriter_fourcc('M','J','P','G'); # See also http://www.fourcc.org/codecs.php vid_fourcc = cv2.VideoWriter_fourcc(*'XVID') video_writer = cv2.VideoWriter(filename=video_file_name, fourcc=vid_fourcc, fps=10, frameSize=(img.shape[1], img.shape[0])) else: common.DebugPrint("Error in creating video writer") sys.exit(1) video_writer.write(img) video_writer.release() common.DebugPrint("Finished writing the video") return
def open_video_capture(video_file_name, video_type): """ Unfortunately, normally cv2.VideoCapture() continues even if it does not find videoPathFileName """ assert os.path.isfile(video_file_name) # From http://docs.opencv.org/modules/highgui/doc/reading_and_writing_images_and_video.html#videocapture-videocapture capture = cv2.VideoCapture(video_file_name) # Inspired from https://stackoverflow.com/questions/16703345/how-can-i-use-opencv-python-to-read-a-video-file-without-looping-mac-os if config.OCV_OLD_PY_BINDINGS: frame_count = int(capture.get(cv2.cv.CV_CAP_PROP_FRAME_COUNT)) else: frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT)) if config.OCV_OLD_PY_BINDINGS: capture.set(cv2.cv.CV_CAP_PROP_POS_FRAMES, config.initFrame[video_type]) else: capture.set(cv2.CAP_PROP_POS_FRAMES, config.initFrame[video_type]) if config.OCV_OLD_PY_BINDINGS: width = capture.get(cv2.cv.CV_CAP_PROP_FRAME_WIDTH) height = capture.get(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT) fps = capture.get(cv2.cv.CV_CAP_PROP_FPS) codec = capture.get(cv2.cv.CV_CAP_PROP_FOURCC) else: width = capture.get(cv2.CAP_PROP_FRAME_WIDTH) height = capture.get(cv2.CAP_PROP_FRAME_HEIGHT) fps = capture.get(cv2.CAP_PROP_FPS) codec = capture.get(cv2.CAP_PROP_FPS) assert width < 32767 # we use np.int16 assert height < 32767 # we use np.int16 duration = frame_count / fps print("Video '%s' has resolution %dx%d, %.2f fps and " "%d frames, duration %.2f secs, codec=%s" % (video_file_name, width, height, fps, frame_count, duration, codec)) steps = [config.counterQStep, config.counterRStep] # TODO: take into account also initFrame used_frame_count = frame_count / steps[video_type] assert not ((video_type == 0) and (used_frame_count <= 10)) common.DebugPrint( "We use video '%s', with %d frames, from which we use ONLY %d\n" % (video_file_name, frame_count, used_frame_count)) resolution = (width, height) return capture, frame_count, resolution
def testSpatialAlignmentEvangelidis(self): """ We test Evangelidis Spatial Alignment, given the precomputed temporal alignment, the cross array. """ videoPathFileNameQ = "Videos/input.avi" videoPathFileNameR = "Videos/reference.avi" """ We want to use the same JPEGs Matlab is using - since the algorithm is somewhat sensitive to quantisation errors. """ assert config.TESTING_IDENTICAL_MATLAB == True import ReadVideo captureQ, frameCountQ, resVidQ = ReadVideo.OpenVideoCapture( videoPathFileNameQ, 0) common.DebugPrint("Alex: frameCountQ = %d" % frameCountQ) captureR, frameCountR, resVidR = ReadVideo.OpenVideoCapture( videoPathFileNameR, 1) common.DebugPrint("Alex: frameCountR = %d" % frameCountR) """ The cross result we obtain for the videos from Evangelidis, with step=25 (1fps). """ crossref = [[0, 48], [1, 48], [2, 48], [3, 48], [4, 48], [5, 48], [6, 0], [7, 0], [8, 60], [9, 60], [10, 60], [11, 60], [12, 67], [13, 67], [14, 67], [15, 67], [16, 67], [17, 67], [18, 72], [19, 72], [20, 72], [21, 78], [22, 78], [23, 78], [24, 78], [25, 78], [26, 82], [27, 82], [28, 82], [29, 54], [30, 54]] crossref = np.array(crossref) #SpatialAlignmentEvangelidis(cross, captureQ=None, captureR=None); res = SpatialAlignmentEvangelidis(crossref, captureQ, captureR) common.DebugPrint("Corrected cross from SpatialAlignmentEvangelidis() = %s" % \ str(res)) """
def spatial_consistency(space_xy, qcen, M, st_threshold, cropflag): ordvot = np.zeros((M, 1)) for iii in range(1, M + 1): #uu=space_xy(:,(iii-1)*2+1:(iii-1)*2+2); uu = space_xy[:, (iii - 1) * 2:(iii - 1) * 2 + 1] #pn=~isnan(uu(:,1)); pn = np.logical_not(np.isnan(uu[:, 0])) if isempty(pn): continue #vv=(uu(pn,:)-qcen(pn,:)); vv = (uu[pn, :] - qcen[pn, :]) if cropflag == 1: #inliers=(sign(vv(:,2))==-1)&(abs(vv(:,1))<10); inliers = (sign(vv[:, 1]) == -1) and (abs(vv[:, 0]) < 10) #outliers = (sign(vv(:,2))==1)|(abs(vv(:,1))>10) outliers = (sign(vv[:, 1]) == 1) or (abs(vv[:, 0]) > 10) bifla = sum(inliers) #/(sum(outliers)+sum(inliers)) if isnan(bifla): ordvot[iii - 1] = 0 elif isinf(bifla): ordvot[iii - 1] = inliers.sum() else: ordvot[iii - 1] = bifla else: #inliers=sqrt(sum(vv.^2,2))<st_threshold; inliers = sqrt((vv**2).sum(1)) < st_threshold #outliers = sqrt(sum(vv.^2,2)) > st_threshold outliers = sqrt((vv**2).sum(1)) > st_threshold bifla = inliers.sum() / (outliers.sum() + inliers.sum()) if isnan(bifla): ordvot[iii - 1] = 0 elif isinf(bifla): ordvot[iii - 1] = inliers.sum() else: ordvot[iii - 1] = bifla if common.MY_DEBUG_STDOUT: common.DebugPrint("ordvot = %s" % str(ordvot)) return ordvot
def simulated_annealing(): L = GetALength() common.DebugPrint("simulated_annealing(): L = %s" % str(L)) #L = len(A) x0 = random.choice(xrange(0, L)) T = 1. k = 1 x = x0 x_best = x0 # Alex: #while T > 1e-3: #while T > 1e-1: # 901 iterations #while T > 1e-2: # 991 iterations #while T > 5e-1: # 501 iterations #while T > 6e-1: # 401 iterations #while T > 7e-1: # 301 iterations #while T > 8e-1: # 201 iterations #while T > 9.5e-1: # 51 iterations - suboptimal (401, 228) instead of (400, 377) #while T > 9.7e-1: # 31 iterations - suboptimal (402, 198) instead of (400, 377) """ In a run obtained suboptimal (411, 78) instead of (400, 377), worse than 51 iterations In another run obtained optimal (400, 377) result. For our example from test.py this one converges nice 50% of the times. """ while T > 9e-1: # 101 iterations: x = make_move(x, T) if GetA(x) < GetA(x_best): x_best = x T = update_temperature(T, k) k += 1 common.DebugPrint("iterations: %d" % k) return x, x_best, x0
def Benchmark(): global captureQ, frameCountQ global captureR, frameCountR # Doing benchmarking while True: if config.OCV_OLD_PY_BINDINGS: frame1 = captureQ.get(cv2.cv.CV_CAP_PROP_POS_FRAMES) else: frame1 = captureQ.get(cv2.CAP_PROP_POS_FRAMES) common.DebugPrint("Alex: frame1 = %d" % frame1) MatchFrames.counterQ = int(frame1) #0 common.DebugPrint("Alex: MatchFrames.counterQ = %d" % MatchFrames.counterQ) retQ, imgQ = captureQ.read() if retQ == False: break if False: grayQ = common.ConvertImgToGrayscale(imgQ) MatchFrames.Main_img1(imgQ, MatchFrames.counterQ) # 36.2 secs (38.5 secs with Convert to RGB) common.DebugPrint("Alex: time after Feature Extraction of all frames of " \ "video 1 = %s" % GetCurrentDateTimeStringWithMilliseconds()) while True: if config.OCV_OLD_PY_BINDINGS: frameR = captureR.get(cv2.cv.CV_CAP_PROP_POS_FRAMES) else: frameR = captureR.get(cv2.CAP_PROP_POS_FRAMES) common.DebugPrint("Alex: frameR = %d" % frameR) MatchFrames.counterR = int(frameR) #0 common.DebugPrint("Alex: counterR = %d" % (MatchFrames.counterR)) retR, imgR = captureR.read() if retR == False: break if False: grayR = common.ConvertImgToGrayscale(imgR) MatchFrames.Main_img2(imgR, MatchFrames.counterR) # Note: 47.2 secs (56.7 secs with Convert to RGB) common.DebugPrint("Alex: time after Feature Extraction of all frames of " \ "video 2 and (FLANN?) matching once for each frame = %s" % \ GetCurrentDateTimeStringWithMilliseconds()) quit()
def compute_features_1(a_img_1, a_counter_q): global kp1, desc1 global counter_q global img_q img_q = a_img_1 counter_q = a_counter_q common.DebugPrint("compute_features_1(): counter_q = %d" % counter_q) if kp1[counter_q] is None: t1 = float(cv2.getTickCount()) kp1[counter_q], desc1[counter_q] = detector.detectAndCompute( a_img_1, None) t2 = float(cv2.getTickCount()) my_time = (t2 - t1) / cv2.getTickFrequency() common.DebugPrint( "compute_features_1(): detector.detectAndCompute() took %.6f [sec]" % my_time) common.DebugPrint("a_img_1 - %d features" % len(kp1[counter_q]))
def cluster_unmatched_keypoints(z): # Start time profiling for the inner loop t1 = float(cv2.getTickCount()) z = np.array(z) # convert to np.float32 z = np.float32(z) n = len(z) # the number of points common.DebugPrint("N = %d" % n) # TODO: should we cluster also the non-matched features of the reference # video (B) ? # TODO:_IMPORTANT: now Hierarchical Clustering returns the clusters - # you can color them differently z = Clustering.hierarchical_clustering(z, n) t2 = float(cv2.getTickCount()) my_time = (t2 - t1) / cv2.getTickFrequency() common.DebugPrint( "cluster_unmatched_keypoints(): HierarchicalClustering() " "took %.5f [sec]" % my_time) return z
def benchmark(): global captureQ, frameCountQ global captureR, frameCountR # Doing benchmarking while True: if config.OCV_OLD_PY_BINDINGS: frame1 = captureQ.get(cv2.cv.CV_CAP_PROP_POS_FRAMES) else: frame1 = captureQ.get(cv2.CAP_PROP_POS_FRAMES) common.DebugPrint("Alex: frame1 = %d" % frame1) MatchFrames.counter_q = int(frame1) common.DebugPrint("Alex: MatchFrames.counter_q = %d" % MatchFrames.counter_q) ret_q, img_q = captureQ.read() if not ret_q: break # TODO: This method below doesn't seem to exist? # MatchFrames.Main_img1(img_q, MatchFrames.counter_q) # 36.2 secs (38.5 secs with Convert to RGB) common.DebugPrint("Alex: time after Feature Extraction of all frames of " "video 1 = %s" % common.GetCurrentDateTimeStringWithMilliseconds()) while True: if config.OCV_OLD_PY_BINDINGS: frame_r = captureR.get(cv2.cv.CV_CAP_PROP_POS_FRAMES) else: frame_r = captureR.get(cv2.CAP_PROP_POS_FRAMES) common.DebugPrint("Alex: frame_r = %d" % frame_r) MatchFrames.counter_r = int(frame_r) common.DebugPrint("Alex: counter_r = %d" % MatchFrames.counter_r) ret_r, img_r = captureR.read() if not ret_r: break # TODO: This method below doesn't seem to exist? # MatchFrames.Main_img2(img_r, MatchFrames.counter_r) # Note: 47.2 secs (56.7 secs with Convert to RGB) common.DebugPrint( "Alex: time after Feature Extraction of all frames of " "video 2 and (FLANN?) matching once for each frame = %s" % common.GetCurrentDateTimeStringWithMilliseconds()) quit()
def GetA(aCounter2): global capture2 # We get the frame at each access and convert it to RGB - we don't store the result in memory # We memoize the value corresponding to aCounter2 in Acache[aCounter2], in the idea that Simulated Annealing might come back to this value if aCounter2 in Acache: return Acache[aCounter2] # If we try to seek to a frame out-of-bounds frame it gets to the last one capture2.set(cv2.cv.CV_CAP_PROP_POS_FRAMES, aCounter2) frame2 = capture2.get(cv2.cv.CV_CAP_PROP_POS_FRAMES) if False: print "Alex: frame2 = %d" % frame2 #aCounter2 = int(frame2) #0 assert aCounter2 == int(frame2) #0 ret2, img2 = capture2.read() if (ret2 == False) or (GetALength() <= aCounter2): common.DebugPrint( "Get(): REALLY BAD error: aCounter2 = %d, ret2 = %s" % \ (aCounter2, str(ret2))) quit() #break if False: # Note: img1 and img2 are RGB-images. I need to convert them to grayscale (as cv2.imread(fn1, 0) ) gray2 = ConvertImgToGrayscale(img2) res = MatchFrames.Main_img2(img2, aCounter2) #print "GetA(): res = %s" % (str(res)) # We memoize this result - The ReadVideo module is responsible to reset this dictionary for a new processing Acache[aCounter2] = -res[1] # We return the number of matches return -res[1]
def pre_main(n_frames_q, n_frames_r): global detector, matcher global num_frames_q, num_frames_r global kp1, kp2 global desc1, desc2 """ # From http://docs.scipy.org/doc/numpy/reference/generated/numpy.set_printoptions.html numpy.set_printoptions(precision=None, threshold=None, edgeitems=None, linewidth=None, suppress=None, nanstr=None, infstr=None, formatter=None) threshold : int, optional Total number of array elements which trigger summarization rather than full repr (default 1000). suppress : bool, optional Whether or not suppress printing of small floating point values using scientific notation (default False). """ np.set_printoptions(threshold=1000000, linewidth=3000) num_frames_q = n_frames_q num_frames_r = n_frames_r feature_name = config.FEATURE_DETECTOR_AND_MATCHER common.DebugPrint("Alex: feature_name = %s" % feature_name) detector, matcher = init_feature(feature_name) if detector is not None: print("using %s" % feature_name) else: print("unknown feature: %s" % feature_name) sys.exit(1) kp1 = [None] * num_frames_q kp2 = [None] * num_frames_r desc1 = [None] * num_frames_q desc2 = [None] * num_frames_r
scipy.show_config(); # See http://docs.scipy.org/doc/numpy/reference/generated/numpy.set_printoptions.html # We use 7 digits precision and suppress using scientific notation. np.set_printoptions(precision=7, suppress=True, \ threshold=70000, linewidth=4000); #threshold=7000000, linewidth=4000); #threshold=7000, linewidth=300); #threshold=1000000, linewidth=3000); # Inspired from \OpenCV2-Python-Tutorials-master\source\py_tutorials\py_core\py_optimization # normally returns True - relates to using the SIMD extensions of x86: SSX, AVX common.DebugPrint("cv2.useOptimized() is %s" % str(cv2.useOptimized())); if False: cv2.setUseOptimized(True); cv2.useOptimized(); """ From http://docs.opencv.org/modules/core/doc/utility_and_system_functions_and_macros.html#checkhardwaresupport CV_CPU_MMX - MMX CV_CPU_SSE - SSE CV_CPU_SSE2 - SSE 2 CV_CPU_SSE3 - SSE 3 CV_CPU_SSSE3 - SSSE 3 CV_CPU_SSE4_1 - SSE 4.1 CV_CPU_SSE4_2 - SSE 4.2 CV_CPU_POPCNT - POPCOUNT
def multiscale_quad_tree(r_path, threshold, scale_index): common.DebugPrint("Entered multiscale_quad_tree(scale_index=%d)" % scale_index) t1 = float(cv2.getTickCount()) found_files = False try: all_quads = np.load("all_quads%d.npz" % scale_index)['arr_0'] all_id = np.load("all_id%d.npz" % scale_index)['arr_0'] all_cen = np.load("all_cen%d.npz" % scale_index)['arr_0'] all_max = np.load("all_max%d.npz" % scale_index)['arr_0'] all_ori = np.load("all_ori%d.npz" % scale_index)['arr_0'] n_d = np.load("n_d%d.npz" % scale_index)['arr_0'] found_files = True common.DebugPrint("multiscale_quad_tree(): loaded the NPZ files for " "this scale.") print( "multiscale_quad_tree(scale_index=%d): time before starting the " "processing for the given query video = %s" % scale_index, common.GetCurrentDateTimeStringWithMilliseconds()) except: common.DebugPrintErrorTrace() if not found_files: rd = r_path # OpenCV's KD-tree implementation does NOT accept float64 all_quads = np.array([]).astype(np.float32) all_cen = np.array([]).astype(np.float32) all_id = np.array([]).astype(np.float32) all_max = np.array([]).astype(np.float32) all_ori = np.array([]).astype(np.float32) n_d = np.zeros((len(rd), 1)) if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_tree(): n_d.shape = " "%s" % str(n_d.shape)) common.DebugPrint("multiscale_quad_tree(): n_d = %s" % str(n_d)) # Alex: for each reference video frame we compute the quads for iFor in range(len(rd)): # Alex: IMPORTANT: This loads into pp the multiscale Harris feature # saved in file r_path+rd(i).name # load harris locations of image (already computed) pp = r_path[iFor] if iFor % 10 == 0: common.DebugPrint("multiscale_quad_tree(): iFor = %d" % iFor) common.DebugPrint("multiscale_quad_tree(): scale_index = " "%s" % str(scale_index)) common.DebugPrint("multiscale_quad_tree(): threshold = " "%s" % str(threshold)) # Alex: We put in points the rows of pp having the # 3rd element == scale_index, and we take out the 3rd column of pp # [out,cen,maxdis,ori]=findquads(pp(pp(:,3)==scale_index,1:2),threshold,1) points = pp[pp[:, 2] == scale_index, 0:2] out, cen, maxdis, ori = findquads.findquads(points, threshold, 1) n_d[iFor] = out.shape[0] temp = np.zeros((out.shape[0], 1)) + iFor if common.MY_DEBUG_STDOUT: common.DebugPrint("Initially:") common.DebugPrint( " multiscale_quad_tree(): all_quads.shape = %s" % str(all_quads.shape)) common.DebugPrint(" multiscale_quad_tree(): all_quads = %s" % str(all_quads)) common.DebugPrint( " multiscale_quad_tree(): all_max.shape = %s" % str(all_max.shape)) common.DebugPrint(" multiscale_quad_tree(): all_max = %s" % str(all_max)) common.DebugPrint( " multiscale_quad_tree(): maxdis.shape = %s" % str(maxdis.shape)) common.DebugPrint(" multiscale_quad_tree(): maxdis = %s" % str(maxdis)) common.DebugPrint( " multiscale_quad_tree(): all_ori.shape = %s" % str(all_ori.shape)) common.DebugPrint(" multiscale_quad_tree(): ori.shape = %s" % str(ori.shape)) common.DebugPrint(" multiscale_quad_tree(): ori = %s" % str(ori)) common.DebugPrint( " multiscale_quad_tree(): all_cen.shape = %s" % str(all_cen.shape)) common.DebugPrint(" multiscale_quad_tree(): all_cen = %s" % str(all_cen)) common.DebugPrint(" multiscale_quad_tree(): cen.shape = %s" % str(cen.shape)) common.DebugPrint(" multiscale_quad_tree(): cen = %s" % str(cen)) common.DebugPrint(" multiscale_quad_tree(): out.shape = %s" % str(out.shape)) common.DebugPrint(" multiscale_quad_tree(): out = %s" % str(out)) if out.size == 0: assert (cen.size == 0) assert (maxdis.size == 0) assert (ori.size == 0) continue """ It crashes at all_quads = np.r_[all_quads, out] with "ValueError: array dimensions must agree except for d_0" because: multiscale_quad_tree(): out = [] multiscale_quad_tree(): out.shape = (2, 0) """ if all_quads.size == 0: all_quads = out.copy() else: all_quads = np.r_[all_quads, out] if all_cen.size == 0: all_cen = cen.copy() else: all_cen = np.r_[all_cen, cen] if all_id.size == 0: all_id = temp.copy() else: all_id = np.r_[all_id, temp] if all_max.size == 0: all_max = maxdis.copy() else: all_max = np.r_[all_max, maxdis] if all_ori.size == 0: all_ori = ori.copy() else: all_ori = np.r_[all_ori, ori] try: np.savez_compressed("all_quads%d" % scale_index, all_quads) np.savez_compressed("all_id%d" % scale_index, all_id) np.savez_compressed("all_cen%d" % scale_index, all_cen) np.savez_compressed("all_max%d" % scale_index, all_max) np.savez_compressed("all_ori%d" % scale_index, all_ori) np.savez_compressed("n_d%d" % scale_index, n_d) except: common.DebugPrintErrorTrace() if all_quads.size == 0: t2 = float(cv2.getTickCount()) my_time = (t2 - t1) / cv2.getTickFrequency() common.DebugPrint("multiscale_quad_tree() " "took %.6f [sec]" % my_time) return None, all_id, all_cen, all_max, all_ori, n_d, all_quads t_build1 = float(cv2.getTickCount()) if config.KDTREE_IMPLEMENTATION == 0: tree = spatial.KDTree(all_quads) elif config.KDTREE_IMPLEMENTATION == 1: # TODO: try to use exact NN-search for the kd-tree - # see http://docs.opencv.org/trunk/modules/flann/doc/flann_fast_approximate_nearest_neighbor_search.html all_quads = all_quads.astype(np.float32) tree = cv2.flann_Index(features=all_quads, params=config.FLANN_PARAMS) t_build2 = float(cv2.getTickCount()) my_time_build = (t_build2 - t_build1) / cv2.getTickFrequency() print("multiscale_quad_tree(): KD-tree build " "took %.6f [sec]" % my_time_build) if common.MY_DEBUG_STDOUT: common.DebugPrint("At the end:") common.DebugPrint(" multiscale_quad_tree(): all_id.shape = %s" % str(all_id.shape)) common.DebugPrint(" multiscale_quad_tree(): all_id = %s" % str(all_id)) common.DebugPrint(" multiscale_quad_tree(): all_cen.shape = %s" % str(all_cen.shape)) common.DebugPrint(" multiscale_quad_tree(): all_cen = %s" % str(all_cen)) common.DebugPrint(" multiscale_quad_tree(): all_max.shape = %s" % str(all_max.shape)) common.DebugPrint(" multiscale_quad_tree(): all_max = %s" % str(all_max)) common.DebugPrint(" multiscale_quad_tree(): all_ori.shape = %s" % str(all_ori.shape)) common.DebugPrint(" multiscale_quad_tree(): all_ori = %s" % str(all_ori)) common.DebugPrint(" multiscale_quad_tree(): n_d.shape = %s" % str(n_d.shape)) common.DebugPrint(" multiscale_quad_tree(): n_d = %s" % str(n_d)) common.DebugPrint(" multiscale_quad_tree(): all_quads.shape = %s" % str(all_quads.shape)) common.DebugPrint(" multiscale_quad_tree(): all_quads = %s" % str(all_quads)) common.DebugPrint(" multiscale_quad_tree(): all_quads.shape before " "kd-tree = %s" % str(all_quads.shape)) try: common.DebugPrint( "multiscale_quad_tree(): sys.getsizeof(tree) = %s" % str(sys.getsizeof(tree))) except: pass common.DebugPrintErrorTrace() t2 = float(cv2.getTickCount()) my_time = (t2 - t1) / cv2.getTickFrequency() print("multiscale_quad_tree() took %.6f [sec]" % my_time) return tree, all_id, all_cen, all_max, all_ori, n_d, all_quads
def main(argv): assert len(sys.argv) >= 3 if FLAGS.preprocess_ref: config.PREPROCESS_REFERENCE_VIDEO_ONLY = True elif FLAGS.process_query_and_align_videos: config.PREPROCESS_REFERENCE_VIDEO_ONLY = False else: config.PREPROCESS_REFERENCE_VIDEO_ONLY = False print("config.PREPROCESS_REFERENCE_VIDEO_ONLY = %s" % str( config.PREPROCESS_REFERENCE_VIDEO_ONLY)) ask_first() # Inspired from https://stackoverflow.com/questions/1520234/how-to-check-which-version-of-numpy-im-using print("numpy.version.version = %s" % str(np.version.version)) print("scipy.version.version = %s" % str(scipy.version.version)) np.show_config() scipy.show_config() # See http://docs.scipy.org/doc/numpy/reference/generated/numpy.set_printoptions.html # We use 7 digits precision and suppress using scientific notation. np.set_printoptions(precision=7, suppress=True, threshold=70000, linewidth=4000) # Inspired from \OpenCV2-Python-Tutorials-master\source\py_tutorials\py_core\py_optimization # normally returns True - relates to using the SIMD extensions of x86: # SSX, AVX common.DebugPrint("cv2.useOptimized() is %s" % str(cv2.useOptimized())) """ From http://docs.opencv.org/modules/core/doc/utility_and_system_functions_and_macros.html#checkhardwaresupport CV_CPU_MMX - MMX CV_CPU_SSE - SSE CV_CPU_SSE2 - SSE 2 CV_CPU_SSE3 - SSE 3 CV_CPU_SSSE3 - SSSE 3 CV_CPU_SSE4_1 - SSE 4.1 CV_CPU_SSE4_2 - SSE 4.2 CV_CPU_POPCNT - POPCOUNT CV_CPU_AVX - AVX """ # TODO: Figure out the correct way to reference these in OpenCV 3.x """ # Need to setUseOptimized before calling checkHardwareSupport cv2.setUseOptimized(True) if config.OCV_OLD_PY_BINDINGS == False: featDict = {cv2.CpuFeatures.CV_CPU_AVX: "AVX", cv2.CPU_MMX: "MMX", cv2.CPU_NEON: "NEON", cv2.CPU_POPCNT: "POPCNT", cv2.CPU_SSE: "SSE", cv2.CPU_SSE2: "SSE2", cv2.CPU_SSE3: "SSE3", cv2.CPU_SSE4_1: "SSE4.1", cv2.CPU_SSE4_2: "SSE4.2", cv2.CPU_SSSE3: "SSSE3"} for feat in featDict: res = cv2.checkHardwareSupport(feat) print("%s = %d" % (featDict[feat], res)) """ # "Returns the number of logical CPUs available for the process." common.DebugPrint("cv2.getNumberOfCPUs() (#logical CPUs) is %s" % str( cv2.getNumberOfCPUs())) common.DebugPrint( "cv2.getTickFrequency() is %s" % str(cv2.getTickFrequency())) video_file_q = sys.argv[1] # input/current video video_file_r = sys.argv[2] # reference video # TODO: use getopt() to run Evangelidis' or "Alex's" algorithm, etc ReadVideo.main(video_file_q, video_file_r)
def MyImageReadMSH(index): global g """ common.DebugPrint("MyImageReadMSH(): initFrame = %d)" % \ (config.initFrame[g.indexVideo])); index += config.initFrame[g.indexVideo]; """ """ common.DebugPrint("Entered MyImageReadMSH(capture=%s, index=%s)" % \ (str(capture), str(index))); """ common.DebugPrint("Entered MyImageReadMSH(index=%s)" % \ (str(index))) """ We must reopen the capture device in each different process, otherwise the program blocks at the first operation in the "global" capture device. """ if g.harlocsFolder.endswith(config.HARRIS_QUERY_FOLDER_NAME): if g.captureQ == None: capture = cv2.VideoCapture(sys.argv[1]) g.captureQ = capture common.DebugPrint("MyImageReadMSH(): new capture=%s" % \ (str(capture))) else: capture = g.captureQ elif g.harlocsFolder.endswith(config.HARRIS_REFERENCE_FOLDER_NAME): if g.captureR == None: capture = cv2.VideoCapture(sys.argv[2]) g.captureR = capture common.DebugPrint("MyImageReadMSH(): new capture=%s" % \ (str(capture))) else: capture = g.captureR else: assert False assert (g.indexVideo == 0) or (g.indexVideo == 1) if config.OCV_OLD_PY_BINDINGS: capture.set(cv2.cv.CV_CAP_PROP_POS_FRAMES, \ index) else: """ From http://docs.opencv.org/modules/highgui/doc/reading_and_writing_images_and_video.html#videocapture-get: <<CV_CAP_PROP_POS_FRAMES 0-based index of the frame to be decoded/captured next.>> """ capture.set(cv2.CAP_PROP_POS_FRAMES, \ index) common.DebugPrint("MyImageReadMSH(): after capture.set()") # This is only for (paranoid) testing purposes: if config.OCV_OLD_PY_BINDINGS: indexCrt = capture.get(cv2.cv.CV_CAP_PROP_POS_FRAMES) else: """ From http://docs.opencv.org/modules/highgui/doc/reading_and_writing_images_and_video.html#videocapture-get: <<CV_CAP_PROP_POS_FRAMES 0-based index of the frame to be decoded/captured next.>> """ indexCrt = capture.get(cv2.CAP_PROP_POS_FRAMES) #assert int(indexCrt) == index; #!!!!TODO: think if OK if int(indexCrt) != index: common.DebugPrint( \ "MyImageReadMSH(): indexCrt != index --> returning black frame") ret = False else: #common.DebugPrint("Alex: frameR = %d" % frameR); #if myIndex > numFramesR: # break; #ret, img = r_path.read(); ret, img = capture.read() #if ret == False: # break; #!!!!TODO: think if well #assert ret == True; if ret == False: common.DebugPrint( "MyImageReadMSH(index=%d): ret == False --> returning None" % index) img = None else: common.DebugPrint("MyImageReadMSH(): img.shape = %s" % str(img.shape)) common.DebugPrint("MyImageReadMSH(): img.dtype = %s" % str(img.dtype)) #!!!!TODO: I suggest to do the gray conversion at reading, not in multi_scale_harris.py if False: # In the Matlab code he reads gray/8bpp JPEGs imgGray = common.ConvertImgToGrayscale(img) if config.VIDEO_FRAME_RESIZE_SCALING_FACTOR != 1: # We resize the image img = Matlab.imresize(img, \ scale=config.VIDEO_FRAME_RESIZE_SCALING_FACTOR) common.DebugPrint("Exiting MyImageReadMSH()") if False: return imgGray return img
def testSynchro(self): common.MY_DEBUG_STDOUT = True multiscale_quad_retrieval.DBGPRINT = True #assert config.temporalDecisionType == 1; #1 """ # 0 --> global solution (multi-scale dynamic programming - the dp3() function) # 1 --> causal (local) syncronization """ if True: # We use dp_Alex (instead of causal, normally) config.temporalDecisionType = 0 #dp3Orig = dp3; multiscale_synchro_decision.dp3 = multiscale_synchro_decision.dp_Alex config.KDTREE_IMPLEMENTATION = 1 # Use OpenCV's KDtree config.FLANN_PARAMS = config.FLANN_PARAMS_DETERMINISTIC """ The cross result from Matlab, for the videos from Evangelidis, with step=25 (1fps), when applying the ORIGINAL causal() temporal alignment. """ resCrossref = [ \ [1001, 2049], [1002, 2049], [1003, 2049], [1004, 2049], [1005, 2049], [1006, 2049], [1007, 2001], [1008, 2001], [1009, 2061], [1010, 2061], [1011, 2061], [1012, 2061], [1013, 2068], [1014, 2068], [1015, 2068], [1016, 2068], [1017, 2068], [1018, 2068], [1019, 2073], [1020, 2073], [1021, 2073], [1022, 2079], [1023, 2079], [1024, 2079], [1025, 2079], [1026, 2079], [1027, 2083], [1028, 2083], [1029, 2083], [1030, 2055], [1031, 2055]] resCrossref = np.array(resCrossref) resCrossref[:, 0] -= 1001 resCrossref[:, 1] -= 2001 #common.DebugPrint("resCrossref = %s" % str(resCrossref)); #crossref = TemporalAlignment(None, None); #crossref = QuadTreeDecision(None, None); crossref = QuadTreeDecision() common.DebugPrint("crossref = %s" % str(crossref)) aZero = resCrossref - crossref common.DebugPrint("aZero = %s" % str(aZero)) """ Note: if we use the OpenCV KD-tree implementation in findquads (instead of SciPy implementation) we will have different results on rows 5 and 6. """ if config.KDTREE_IMPLEMENTATION == 1: common.DebugPrint("The following 2 values we disregard:") common.DebugPrint(" aZero[5, 1] = %s" % str(aZero[5, 1])) common.DebugPrint(" aZero[6, 1] = %s" % str(aZero[6, 1])) aZero[5, 1] = 0 aZero[6, 1] = 0 self.assertTrue((aZero == 0).all())
def ComputeHarlocs(capture, counterStep, folderName, fileNamePrefix, fileNameExtension=".csv", indexVideo=-1): print( \ "Entered ComputeHarlocs(capture=%s, counterStep=%d, folderName=%s, " \ "indexVideo=%d)" % \ (str(capture), counterStep, folderName, indexVideo)) harlocsFolder = config.VIDEOS_FOLDER + "/" + folderName t1 = float(cv2.getTickCount()) harlocs = [] if config.OCV_OLD_PY_BINDINGS: numFrames = int(capture.get(cv2.cv.CV_CAP_PROP_FRAME_COUNT)) else: numFrames = int(capture.get(cv2.CAP_PROP_FRAME_COUNT)) common.DebugPrint("ComputeHarlocs(): numFrames = %d" % numFrames) if not os.path.exists(harlocsFolder): os.makedirs(harlocsFolder) else: #!!!!TODO: check that the loaded Harlocs are complete - same frame numbers as in the videos # Folder with precomputed Harris features exists folderContent = os.listdir(harlocsFolder) sortedFolderContent = sorted(folderContent) for fileName in sortedFolderContent: pathFileName = harlocsFolder + "/" + fileName """ common.DebugPrint("ComputeHarlocs(): pathFileName = %s" % pathFileName); common.DebugPrint("ComputeHarlocs(): fileName = %s" % fileName); """ if os.path.isfile(pathFileName) and \ fileName.startswith(fileNamePrefix) and \ pathFileName.endswith(fileNameExtension): common.DebugPrint("ComputeHarlocs(): Loading %s" % pathFileName) harrisFeatures = multi_scale_harris.LoadMultiScaleHarrisFeatures( pathFileName) harlocs.append(harrisFeatures) if config.endFrame[indexVideo] == -1: assert (len(harlocs) + config.initFrame[indexVideo]) == numFrames #!!!!TODO: if condition is NOT met, give a nicer error, or redo computations of Harlocs else: assert (len(harlocs) + config.initFrame[indexVideo] ) == config.endFrame[indexVideo] + 1 #!!!!TODO: if condition is NOT met, give a nicer error, or redo computations of Harlocs return harlocs if config.USE_MULTITHREADING == True: global g g.captureQ = None # We need to reopen the capture device in each process, separately g.captureR = None # We need to reopen the capture device in each process, separately #g.capture = capture; g.harlocsFolder = harlocsFolder g.fileNamePrefix = fileNamePrefix g.fileNameExtension = fileNameExtension g.indexVideo = indexVideo if config.OCV_OLD_PY_BINDINGS: frameCount = int(capture.get(cv2.cv.CV_CAP_PROP_FRAME_COUNT)) else: frameCount = int(capture.get(cv2.CAP_PROP_FRAME_COUNT)) #listParams = range(frameCount); listParams = range(config.initFrame[indexVideo], frameCount, counterStep) common.DebugPrint("ComputeHarlocs(): frameCount = %d" % frameCount) print("ComputeHarlocs(): Spawning a pool of %d workers" % \ config.numProcesses) """ # DEBUG purposes ONLY - since when we use Pool() and call function, if # we have an error in the function the exception reported is very # vague... for i in listParams: IterationStandaloneMSH(i); #import time #time.sleep(1000); """ """ Start worker processes to use on multi-core processor (circumvent also the GIL issue). """ pool = multiprocessing.Pool(processes=config.numProcesses) print("ComputeHarlocs(): Spawned a pool of %d workers" % \ config.numProcesses) #res = pool.map(IterationStandaloneMSH, listParams); # See https://docs.python.org/2/library/multiprocessing.html#module-multiprocessing.pool res = pool.map(func=IterationStandaloneMSH, iterable=listParams, \ chunksize=1) print("Pool.map returns %s" % str(res)) """ From https://medium.com/building-things-on-the-internet/40e9b2b36148 close the pool and wait for the work to finish """ pool.close() pool.join() #!!!!TODO: do more efficient - don't load the results from the CSV files return ComputeHarlocs(capture, counterStep, folderName, \ fileNamePrefix, fileNameExtension, indexVideo) #return []; #indexHarloc = 0; while True: if config.OCV_OLD_PY_BINDINGS: framePos = capture.get(cv2.cv.CV_CAP_PROP_POS_FRAMES) else: """ From http://docs.opencv.org/modules/highgui/doc/reading_and_writing_images_and_video.html#videocapture-get: <<CV_CAP_PROP_POS_FRAMES 0-based index of the frame to be decoded/captured next.>> """ framePos = capture.get(cv2.CAP_PROP_POS_FRAMES) common.DebugPrint("ComputeHarlocs(): framePos = %d" % framePos) counter = int(framePos) #0 common.DebugPrint("ComputeHarlocs(): counter = %d" % counter) ret, img = capture.read() if common.MY_DEBUG_STDOUT: common.DebugPrint("ComputeHarlocs(): img = %s" % str(img)) if False and config.SAVE_FRAMES: fileName = config.IMAGES_FOLDER + "/img_%05d.png" % counter if not os.path.exists(fileName): #print "dir(img) = %s"% str(dir(img)) """ imgCV = cv.fromarray(img) cv2.imwrite(fileName, imgCV) """ cv2.imwrite(fileName, img) #if ret == False: #MatchFrames.counterQ == 3: if (ret == False) or ((counter > numFrames) or \ (config.endFrame[indexVideo] != -1 and \ counter > config.endFrame[indexVideo])): break if config.VIDEO_FRAME_RESIZE_SCALING_FACTOR != 1: img = Matlab.imresize(img, \ scale=config.VIDEO_FRAME_RESIZE_SCALING_FACTOR) im = img pp = multi_scale_harris.multi_scale_harris(im, nos, disp=0) # n=0:nos-1 #harlocs = pp harlocs.append(pp) multi_scale_harris.StoreMultiScaleHarrisFeatures( \ harlocsFolder + "/" + fileNamePrefix + "%05d%s" % \ (counter, fileNameExtension), pp) counter += counterStep # If we try to seek to a frame out-of-bounds frame it gets to the last one if config.OCV_OLD_PY_BINDINGS: capture.set(cv2.cv.CV_CAP_PROP_POS_FRAMES, counter) else: capture.set(cv2.CAP_PROP_POS_FRAMES, counter) #indexHarloc += 1; t2 = float(cv2.getTickCount()) myTime = (t2 - t1) / cv2.getTickFrequency() common.DebugPrint("ComputeHarlocs(): computing the multiscale harlocs " \ "took %.6f [sec]" % myTime) #common.DebugPrint("ComputeHarlocs(): len(harlocs) = %s" % str(len(harlocs))); if False: for i, h in enumerate(harlocs): #common.DebugPrint("ComputeHarlocs(): len(harlocs[%d]) = %d" % \ # (i, len(h))); multi_scale_harris.StoreMultiScaleHarrisFeatures( harlocsFolder + "/" + fileNamePrefix + "%05d.txt" % i, h) #if False: if common.MY_DEBUG_STDOUT: common.DebugPrint("ComputeHarlocs(): harlocs = %s" % str(harlocs)) return harlocs
def output_crossref_images(crossref, capture_q, capture_r): if not os.path.exists(config.FRAME_PAIRS_MATCHES_FOLDER): os.makedirs(config.FRAME_PAIRS_MATCHES_FOLDER) rframe = None gray_rframe = None last_r_idx = None t0 = float(cv2.getTickCount()) for q_idx, r_idx in crossref: q_idx = int(q_idx) r_idx = int(r_idx) if common.MY_DEBUG_STDOUT: t1 = float(cv2.getTickCount()) qframe = MyImageRead(capture_q, q_idx, grayscale=False) gray_qframe = common.ConvertImgToGrayscale(qframe) if r_idx != last_r_idx: rframe = MyImageRead(capture_r, r_idx, grayscale=False) gray_rframe = common.ConvertImgToGrayscale(rframe) last_r_idx = r_idx if common.MY_DEBUG_STDOUT: t2 = float(cv2.getTickCount()) common.DebugPrint( "ecc_homo_spacetime(): grabbing frames took %.6f [sec]" % ((t2 - t1) / cv2.getTickFrequency())) # Default diff is a G-channel swap. gdiff = np.zeros(qframe.shape) gdiff[:, :, 0] = gray_qframe gdiff[:, :, 1] = gray_rframe # G-Channel Swap gdiff[:, :, 2] = gray_qframe if config.VISUAL_DIFF_FRAMES: # Compute difference between frames. diff_mask = cv2.absdiff(qframe, rframe) for i in range(diff_mask.shape[2]): diff_mask[:, :, i] = cv2.GaussianBlur(src=diff_mask[:, :, i], ksize=(7, 7), sigmaX=10) diff_mask_sum = diff_mask.sum(axis=2) r_diff, c_diff = np.nonzero( diff_mask_sum >= config.MEANINGFUL_DIFF_THRESHOLD * diff_mask.shape[2]) meaningful_indices = zip(r_diff, c_diff) # Create all black image so we can set the diff pixels to white. diff_mask = np.zeros((diff_mask.shape[0], diff_mask.shape[1]), dtype=np.uint8) diff_mask[(r_diff, c_diff)] = 255 assert r_diff.size == c_diff.size assert r_diff.size == len(meaningful_indices) res = cv2.findContours(image=diff_mask, mode=cv2.RETR_TREE, method=cv2.CHAIN_APPROX_SIMPLE) contours = res[1] color_c = 255 meaningful_contours = 0 diff_mask = np.zeros((diff_mask.shape[0], diff_mask.shape[1]), dtype=np.uint8) for index_c, contour in enumerate(contours): if len(contour) < 15: continue else: cv2.drawContours(image=diff_mask, contours=[contour], contourIdx=0, color=color_c, thickness=-1) meaningful_contours += 1 cv2.imwrite( config.FRAME_PAIRS_MATCHES_FOLDER + ("%.6d_diff_mask" % q_idx) + config.imformat, diff_mask.astype(int)) if config.SHOW_MASKED_DIFF: qframe_diff = cv2.bitwise_and(qframe, qframe, mask=diff_mask) rframe_diff = cv2.bitwise_and(rframe, rframe, mask=diff_mask) # Save the masked diffs. cv2.imwrite( config.FRAME_PAIRS_MATCHES_FOLDER + ("%.6d_query_diff" % q_idx) + config.imformat, qframe_diff.astype(int)) cv2.imwrite( config.FRAME_PAIRS_MATCHES_FOLDER + ("%.6d_ref_diff" % q_idx) + config.imformat, rframe_diff.astype(int)) if config.SAVE_FRAMES: # We use q_idx because ref video is aligned against the query video. # These frames represent the aligned output. cv2.imwrite( config.FRAME_PAIRS_MATCHES_FOLDER + ("%.6d_query" % q_idx) + config.imformat, qframe.astype(int)) cv2.imwrite( config.FRAME_PAIRS_MATCHES_FOLDER + ("%.6d_ref" % q_idx) + config.imformat, rframe.astype(int)) cv2.imwrite( config.FRAME_PAIRS_MATCHES_FOLDER + ("%.6d_gchan_diff" % q_idx) + config.imformat, gdiff.astype(int)) print("Saving crossref frames took %.6f [sec]" % ((float(cv2.getTickCount()) - t0) / cv2.getTickFrequency()))
def temporal_alignment(counter_q, frame_q, capture_r, num_frames_r, num_features_matched, f_output): global p1, p2, kp_pairs, status, H, nonp1 if config.USE_EXHAUSTIVE_SEARCH: max_features_matched = -2000000000 pos_max_features_matched = -1 while True: if config.OCV_OLD_PY_BINDINGS: frameR = capture_r.get(cv2.cv.CV_CAP_PROP_POS_FRAMES) else: frameR = capture_r.get(cv2.CAP_PROP_POS_FRAMES) common.DebugPrint("Alex: frameR = %d" % frameR) counter_r = int(frameR) ret2, img_r = capture_r.read() if (ret2 == False) or (counter_r > num_frames_r): break if config.SAVE_FRAMES: file_name = config.IMAGES_FOLDER + "/imgR_%05d.png" % counter_r if not os.path.exists(file_name): cv2.imwrite(file_name, img_r) res = compute_features_and_match_2(img_r, counter_r) cost_used = 1 if cost_used == 0: pass else: # myTemporalAlignmentCost is the sum of distances of best-pairs # (closest neighbors) of matched features res = (res[0], -myTemporalAlignmentCost) num_features_matched[counter_q][counter_r] = res[1] compute_best_fast = False if max_features_matched < res[1]: max_features_matched = res[1] pos_max_features_matched = counter_r if config.SAVE_FRAMES: if not compute_best_fast: # TODO: don't do it even for best candidates - # only for the best ONE - this implies redoing probably # some computation from temporal_alignment() for the # best frame pair # We call rest() in order to compute vis and visOrig. rest("Image Match") vis_best = vis.copy() vis_orig_best = vis_orig.copy() else: p1_best = p1 p2_best = p2 kp_pairs_best = kp_pairs status_best = status h_best = H nonp1_best = nonp1 common.DebugPrint("Alex: counter_r = %d" % counter_r) counter_r += config.counterRStep """ If we try to seek to a frame out-of-bounds frame it gets to the last one. """ if config.OCV_OLD_PY_BINDINGS: capture_r.set(cv2.cv.CV_CAP_PROP_POS_FRAMES, counter_r) else: capture_r.set(cv2.CAP_PROP_POS_FRAMES, counter_r) common.DebugPrint( "Alex: Time = %s" % common.GetCurrentDateTimeStringWithMilliseconds()) my_text = " Frame #%d of video A: frame #%d of video B, " \ "with %d features matched (time = %s)" % \ (counter_q, pos_max_features_matched, max_features_matched, common.GetCurrentDateTimeStringWithMilliseconds()) print(my_text, file=f_output) f_output.flush() counter_r_best = pos_max_features_matched else: # We empty the memoization cache before SimAnneal.main() SimAnneal.Acache = {} res = SimAnneal.main() res2 = (res[0], -res[1]) common.DebugPrint( "Best solution for frame counter_q=%d is %s. Time = %s" % (counter_q, str(res2), common.GetCurrentDateTimeStringWithMilliseconds())) # TODO: check if OK: counter_r_best = res[0] if compute_best_fast: # TODO: check that these assignments really refer the objects defined # above (when updating) and that there are no escapes of the # values/objects that result in side-effects updating the respective # objects and messing everything up - better said we look if # redefinitons of the rhs are done inside (some of their subelements) # or totally (reassign a completely NEW object). p1 = p1_best p2 = p2_best kp_pairs = kp_pairs_best status = status_best H = h_best nonp1 = nonp1_best rest("Image Match") vis_best = vis.copy() vis_orig_best = vis_orig.copy() if config.SAVE_FRAMES: """ We display frames imgQ and img_r with features (from temporal) and clusters on them. """ cv2.imwrite( config.FRAME_PAIRS_MATCHES_FOLDER + "/img_proc_%05d_%05d.png" % (counter_q, counter_r_best), vis_best) # We display also the orig frames imgQ and img_r cv2.imwrite( config.FRAME_PAIRS_MATCHES_FOLDER + "/img_proc_%05d_%05d_orig.png" % (counter_q, counter_r_best), vis_orig_best) """ We return the len of the status returned by cv2.findHomography() - this is THE indicator of the match of a pair of frames. NOTE: we don't use the result :))) !!!!TODO: think to take it out """ return res[1] # res
def feature_match_and_homography(): # TODO: memoize (but it's gonna be huge) the pair results of matching - # should reduce by 2 the runtime global p1, p2, kp_pairs, status, H print("Entered feature_match_and_homography()...") # raw_matches still has the same len as desc1[counter_q] AND trainDescriptors t1 = float(cv2.getTickCount()) """ opencv2refman.pdf, Section 7.4., page 429: knnMatch "Finds the k best matches for each descriptor from a query set." "k - Count of best matches found per each query descriptor or less if a query descriptor has less than k possible matches in total." """ raw_matches = matcher.knnMatch(desc1[counter_q], trainDescriptors=desc2[counter_r], k=2) p1, p2, kp_pairs = filter_keypoints_matches(kp1[counter_q], kp2[counter_r], raw_matches) t2 = float(cv2.getTickCount()) my_time = (t2 - t1) / cv2.getTickFrequency() common.DebugPrint("feature_match_and_homography(): knnMatch() and " "filter_keypoints_matches() took %.6f [sec]" % my_time) """ # We cluster & display the non-matched keypoints nonp1 = cluster_unmatched_keypoints(nonp1) """ """ raw_matches is a list of corresponding?? pairs of DMatch for the keypoints obtained with knnMatch() . p1, p2 are only the matched keypoints from the 2 frames kp_pairs is pair of corresponding keypoints, obtained with the list zip operation: cardinality ~=???? the cardinality of kp1[counter_q] and kp2[counter_r] """ if len(p1) >= 4: # p1 and p2 are the matched keypoints """ From Section 6.1, page 391, opencv2refman.pdf: (also http://docs.opencv.org/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.html?highlight=findhomography#findhomography): "Finds a perspective transformation between two planes." "However, if not all of the point pairs (srcPoints i ,:math:dstPoints_i ) fit the rigid perspective transformation (that is, there are some outliers), this initial estimate will be poor." "In this case, you can use one of the two robust methods. Both methods, RANSAC and LMeDS , try many different random subsets of the corresponding point pairs (of four pairs each), estimate the homography matrix using this subset and a simple least-square algorithm, and then compute the quality/goodness of the computed homography (which is the number of inliers for RANSAC or the median re-projection error for LMeDs). The best subset is then used to produce the initial estimate of the homography matrix and the mask of inliers/outliers." """ t1 = float(cv2.getTickCount()) H, status = cv2.findHomography(srcPoints=p1, dstPoints=p2, method=cv2.RANSAC, ransacReprojThreshold=5.0) t2 = float(cv2.getTickCount()) my_time = (t2 - t1) / cv2.getTickFrequency() common.DebugPrint( "feature_match_and_homography(): cv2.findHomography() " "took %.6f [sec]" % my_time) if status is None: status = [] if H == None: print("!!!!!!!!found H None - len(p1) = %d" % len(p1)) H = [] return -1, len(p1) print("%d / %d inliers/matched (len(p1) = %d)" % (np.sum(status), len(status), len(p1))) # Note: H is the homography matrix, a 3x3 matrix. common.DebugPrint( "H, the homography matrix, from cv2.findHomography = %s" % str(H)) common.DebugPrint(" len(H) = %d" % len(H)) # Note that len(status) == len(p1) res = (np.sum(status), len(status)) else: H, status = None, None common.DebugPrint( "%d matches found, not enough for homography estimation" % len(p1)) res = (-1, len(p1)) # The result is related to the homography transformation: status sum and len return res
def hierarchical_clustering_with_cv2_unfinished(z, n): # We choose an ~optimal number of clusters min_validity = 1000000 min_validity_k = -1 for k in range(2, 10 + 1): a = [None] * k """ Inspired a bit from https://www.google-melange.com/gsoc/project/google/gsoc2013/abidrahman2/43002, \source\py_tutorials\py_ml\py_kmeans\py_kmeans_opencv\py_kmeans_opencv.rst """ # Define criteria and apply kmeans() criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 10, 1.0) t1 = float(cv2.getTickCount()) ret, label, center = cv2.kmeans(z, k, criteria, 10, cv2.KMEANS_RANDOM_CENTERS) t2 = float(cv2.getTickCount()) my_time = (t2 - t1) / cv2.getTickFrequency() common.DebugPrint( "HierarchicalClusteringWithCV2_UNFINISHED(): " "cv2.kmeans() took %.5f [sec]" % my_time) common.DebugPrint("ret = %s" % str(ret)) common.DebugPrint("label = %s" % str(label)) common.DebugPrint("center = %s" % str(center)) # Now separate the data, Note the flatten() for i in range(k): a[i] = z[label.ravel() == i] common.DebugPrint("A[0] = %s" % str(a[0])) common.DebugPrint("A[0][:, 0] = %s" % str(a[0][:, 0])) common.DebugPrint("A[0][:, 1] = %s" % str(a[0][:, 1])) """ Following Section 3.2 from http://www.csse.monash.edu.au/~roset/papers/cal99.pdf : See if you have the time, for further ideas: https://stackoverflow.com/questions/15376075/cluster-analysis-in-r-determine-the-optimal-number-of-clusters """ intra = 0 for i in range(k): # Gives exception: "TypeError: only length-1 arrays can be converted to Python scalars" for x in range(len(a[i])): intra += np.square(a[i][x, 0] - center[i, 0]) + \ np.square(a[i][x, 1] - center[i, 1]) intra /= n dist_min = 1000000 for i in range(k): for j in range(i + 1, k): dist = np.square(center[i, 0] - center[j, 0]) + \ np.square(center[i, 1] - center[j, 1]) """ dist = sqr(center[i, 0] - center[j, 0]) + \ sqr(center[i, 1] - center[j, 1]) """ common.DebugPrint("dist = %s" % str(dist)) if dist < dist_min: dist_min = dist inter = dist_min """ We want to minimize intra (clusters be dense) and maximize inter (clusters be distant from one another). """ validity = intra / inter if min_validity > validity: min_validity = validity min_validity_k = k if config.USE_GUI: # We clear the figure and the axes plt.clf() plt.cla() # Plot the data for i in range(k): """ Note: A[0][:,0] (i.e., [:,0] is a numpy-specific "split"-operator, not working for standard Python lists. """ plt.scatter(a[i][:, 0], a[i][:, 1], c=colors[i]) plt.scatter(center[:, 0], center[:, 1], s=80, c="b", marker="s") plt.xlabel( "Height. Also: k=%d, intra=%.1f, inter=%.1f, validity = %.4f" % (k, intra, inter, validity)) plt.ylabel("Weight") plt.show() """ TODO!!!! Implement section 4 from http://www.csse.monash.edu.au/~roset/papers/cal99.pdf: See "when we require the number of clusters to be increased, we split the cluster having maximum variance, so the k-means procedure is given good starting cluster centres." """ # !!!!TODO: .... DO THE IMPLEMENTATION, WHITE BOY common.DebugPrint("IMPORTANT: min_validity_k = %d" % min_validity_k)
def hierarchical_clustering(z, n): n = len(z) common.DebugPrint("hierarchical_clustering(): n = %d" % n) # Note: Z is not standard list, but a numpy array if len(z) < 10: # or Z == []: common.DebugPrint("hierarchical_clustering(): Bailing out of " "hierarchical clustering since too few elements " "provided (and I guess we could have issues)") return [] # Vector of (N choose 2) pairwise Euclidian distances d_sch = sch.distance.pdist(z) d_max = d_sch.max() # This parameter is CRUCIAL for the optimal number of clusters generated # This parameter works better for the videos from Lucian threshold = 0.05 * d_max """ I did not find much information on the linkage matrix (linkage_matrix), but from my understanding it is the direct result of the hierarchical clustering, which is performed by recursively splitting clusters, forming a dendrogram forest of trees (see if you have time https://stackoverflow.com/questions/5461357/hierarchical-k-means-in-opencv-without-knowledge-of-k "a forest of hierarchical clustering trees"). The linkage matrix is stores on each row data for a clustered point: - the last element in the row is the leaf in the dendrogram tree forest the point belongs to. The leaf does not really tell you to which final cluster the point belongs to - (IMPORTANT) for this, we have the function sch.fcluster(). See if you have the time (for some better understanding): https://stackoverflow.com/questions/11917779/how-to-plot-and-annotate-hierarchical-clustering-dendrograms-in-scipy-matplotlib See doc: http://docs.scipy.org/doc/scipy/reference/generated/scipy.cluster.hierarchy.linkage.html#scipy.cluster.hierarchy.linkage See if you have the time: https://stackoverflow.com/questions/16883412/how-do-i-get-the-subtrees-of-dendrogram-made-by-scipy-cluster-hierarchy """ linkage_matrix = sch.linkage(d_sch, "single") common.DebugPrint("linkage_matrix = %s" % str(linkage_matrix)) # Inspired from https://stackoverflow.com/questions/7664826/how-to-get-flat-clustering-corresponding-to-color-clusters-in-the-dendrogram-cre index_cluster = sch.fcluster(linkage_matrix, threshold, "distance") common.DebugPrint("index_cluster = %s" % str(index_cluster)) c_max = -1 # We "truncate" later the ending zeros from num_elems num_elems = [0] * (n + 1) # IMPORTANT: It appears the ids of the clusters start from 1, not 0 for e in index_cluster: # print "e = %s" % str(e) num_elems[e] += 1 if c_max < e: c_max = e # c_max is the MAXIMUM optimal number of clusters after the hierarchical # clustering is performed. common.DebugPrint("c_max (the MAX id of final clusters) = %d" % c_max) num_elems = num_elems[0 : c_max + 1] common.DebugPrint("num_elems = %s" % str(num_elems)) """ # We can also use: num_elems.__delslice__(c_max + 1, len(num_elems)) but it's sort of deprecated - see http://docs.python.org/release/2.5.2/ref/sequence-methods.html """ num_clusters = 0 for e in num_elems: if e != 0: num_clusters += 1 common.DebugPrint("num_clusters (the optimal num of clusters) = %d" % num_clusters) assert num_clusters == c_max num_clusters_above_threshold = 0 for i in range(c_max + 1): if num_elems[i] >= \ config.THRESHOLD_NUM_NONMATCHED_ELEMENTS_IN_CLUSTER: common.DebugPrint("num_elems[%d] = %d" % (i, num_elems[i])) num_clusters_above_threshold += 1 common.DebugPrint("num_clusters_above_threshold = %d" % num_clusters_above_threshold) # TODO: Move this to config? return_only_biggest_cluster = False if return_only_biggest_cluster: # TODO: find biggest cluster - sort them after num_elems, etc res = [] for i in range(n): # We start numbering the clusters from 1. if index_cluster[i] == num_clusters: res.append(z[i]) else: res = {} for i in range(n): if num_elems[index_cluster[i]] >= \ config.THRESHOLD_NUM_NONMATCHED_ELEMENTS_IN_CLUSTER: if index_cluster[i] not in res: res[index_cluster[i]] = [] res[index_cluster[i]].append(z[i]) if config.USE_GUI and config.DISPLAY_PYTHON_CLUSTERING: # We clear the figure and the axes plt.clf() plt.cla() # Plot the data for i in range(n): # index_cluster: # print "Z[i, 0] = %.2f, Z[i, 1] = %.2f" % (Z[i, 0], Z[i, 1]) try: col_cluster = colors[index_cluster[i]] except: # IndexError: list index out of range col_cluster = 2 plt.scatter(z[i, 0], z[i, 1], c=col_cluster) plt.xlabel( "Height. (num_clusters = %d, num_clusters_above_threshold = %d)" % (num_clusters, num_clusters_above_threshold)) plt.ylabel("Weight") # From http://matplotlib.org/api/pyplot_api.html#matplotlib.pyplot.axis v = plt.axis() # We invert the y to have 0 up and x axis to have 0 v = (0, v[1], v[3], 0) plt.axis(v) plt.show() return res
def QuadTreeDecision(): """ global r_path, q_path common.DebugPrint("QuadTreeDecision(): r_path = %s" % r_path); common.DebugPrint("QuadTreeDecision(): q_path = %s" % q_path); """ #global harlocsQ, harlocsR common.DebugPrint("Entered QuadTreeDecision().") totalT1 = float(cv2.getTickCount()) r_quadsTree = None """ Matlab code: www=whos('tree'); if size(www,1)>0 kdtree_delete(tree); clear tree; end """ #clear Votes_space H; Votes_space = None H = None #['arr_0'] if r_quadsTree != None: # TODO: clear tree pass if method == 1: #%% search among all reference quads common.DebugPrint( "\nQuadTreeDecision(): Search among all reference quads...(Tree method)" ) try: crossref = np.load("crossref.npz")['arr_0'] common.DebugPrint( "\nQuadTreeDecision(): Found already precomputed crossref.npz - returning it)" ) return crossref except: common.DebugPrintErrorTrace() #BOV_flag=0; BOV_flag = 0 foundFiles = False try: Votes_space = np.load("Votes_space.npz")['arr_0'] H = np.load("H.npz")['arr_0'] foundFiles = True except: common.DebugPrintErrorTrace() if foundFiles == False: """ Alex: scale s is 1 for original frame resolution and the higher we go we have lower image resolutions (we go higher in the Guassian pyramid I think). """ #for s=1:nos for s in range(1, nos + 1): common.DebugPrint("QuadTreeDecision(): Scale %d" % s) #md_threshold=round(s*100+100^(log(s))); md_threshold = round(s * 100 + pow(100, math.log(s))) #[tree,all_id,all_cen,all_max,all_ori,n_d,all_quads]=multiscale_quad_tree(r_path, md_threshold,s); #tree, all_id, all_cen, all_max, all_ori, n_d, all_quads = multiscale_quad_tree.multiscale_quad_tree(r_path, md_threshold, s) r_quadsTree, all_id, all_cen, all_max, all_ori, n_d, all_quads = \ multiscale_quad_tree.multiscale_quad_tree(harlocsR, \ md_threshold, s) if config.PREPROCESS_REFERENCE_VIDEO_ONLY == True: continue if r_quadsTree == None: continue common.DebugPrint("QuadTreeDecision(): md_threshold = %s" % str(md_threshold)) #[Votes_space(:,:,s),H(:,:,s)]=multiscale_quad_retrieval(tree, r_path, q_path, md_threshold, st_threshold, all_ori, all_id, all_max, all_cen,nos, s, cropflag, sequence); # Votes_space(:,:,s),H(:,:s) =multiscale_quad_retrieval(tree, r_path, q_path, md_threshold, st_threshold, all_ori, all_id, all_max, all_cen, nos, s, cropflag, sequence) #Votes_space[:, :, s - 1], H[:, :,s - 1] = multiscale_quad_retrieval.multiscale_quad_retrieval(r_quadsTree, harlocsR, harlocsQ, md_threshold, st_threshold, all_ori, all_id, all_max, all_cen, nos, s, cropflag, sequence) Votes_space_res, H_res = multiscale_quad_retrieval.multiscale_quad_retrieval(r_quadsTree, \ harlocsR, harlocsQ, md_threshold, \ st_threshold, all_ori, all_id, \ all_max, all_cen, nos, s, cropflag, \ sequence) if Votes_space == None: Votes_space = np.zeros((Votes_space_res.shape[0], Votes_space_res.shape[1], nos)) """ Inspired from https://stackoverflow.com/questions/17559140/matlab-twice-as-fast-as-numpy BUT doesn't help in this case: Votes_space = np.asfortranarray(np.zeros( (Votes_space_res.shape[0], Votes_space_res.shape[1], nos) )); """ if H == None: H = np.zeros((H_res.shape[0], H_res.shape[1], nos), dtype=np.int8) """ Inspired from https://stackoverflow.com/questions/17559140/matlab-twice-as-fast-as-numpy BUT doesn't help in this case: H = np.asfortranarray(np.zeros( (H_res.shape[0], H_res.shape[1], nos) )); """ Votes_space[:, :, s - 1] = Votes_space_res H[:, :, s - 1] = H_res if common.MY_DEBUG_STDOUT: common.DebugPrint("QuadTreeDecision(): For scale %d: " \ "Votes_space_res = %s,\n H_res = %s" % \ (s, str(Votes_space_res), str(H_res))) common.DebugPrint("QuadTreeDecision(): For scale %d: " \ "Votes_space_res.shape = %s,\n H_res.shape = %s" % \ (s, str(Votes_space_res.shape), str(H_res.shape))) #quit(); #kdtree_delete(tree); # TODO: think if want to delete kdtree if config.KDTREE_IMPLEMENTATION == 1: r_quadsTree.release() if config.PREPROCESS_REFERENCE_VIDEO_ONLY == True: common.DebugPrint("QuadTreeDecision(): Exiting program " \ "since we finished preprocessing the reference video") common.DebugPrint("QuadTreeDecision(): time before exit = %s" % \ common.GetCurrentDateTimeStringWithMilliseconds()) return None #quit(); if common.MY_DEBUG_STDOUT: common.DebugPrint("QuadTreeDecision(): Before multiscale_synchro_decision(): " \ "Votes_space = %s,\n H = %s" % (str(Votes_space), str(H))) try: # See http://docs.scipy.org/doc/numpy/reference/generated/numpy.savez.html np.savez_compressed("Votes_space", Votes_space) np.savez_compressed("H", H) except: common.DebugPrintErrorTrace() #q_path = [None] * len(harlocsQ); numFramesQ = len(harlocsQ) #r_path = [None] * len(harlocsR); numFramesR = len(harlocsR) if config.temporalDecisionType == 1: # causal solution - "local" #cross=multiscale_synchro_decision(Votes_space, H, q_path, r_path, BOV_flag, cropflag, const_type); crossref = multiscale_synchro_decision.causal( \ Votes_space, H, numFramesQ, numFramesR, BOV_flag, cropflag, \ const_type) # str=['save ' q_path 'cross_baseline cross']; # eval(str) elif config.temporalDecisionType == 0: # decision (non-causal solution) #[y,x,D,Tback,cross] = dp3(Votes_space, r_path, q_path, BOV_flag); y, x, D, Tback, crossref = multiscale_synchro_decision.dp3( \ Votes_space, numFramesR, numFramesQ, BOV_flag) # str=['save ' q_path 'cross_baseline_dp cross']; # eval(str) else: """ !!!!TODO: implement if useful VD (or BoW) NOTE: see config.py for Evangelidis' comments from email of Apr 14, 2014: Basically he argues that: - the VD method is similar in quality with the full-search VS - BoW is not great. """ assert False # not implemented crossref[:, 1] += config.initFrame[1] #myText = "crossref = \n%s" % crossref; myText = "" for r in range(crossref.shape[0]): myText += " %d %d\n" % (crossref[r][0], crossref[r][1]) fOutput = open("crossref.txt", "wt") fOutput.write(myText) fOutput.close() try: # See http://docs.scipy.org/doc/numpy/reference/generated/numpy.savez.html np.savez_compressed("crossref", crossref) except: common.DebugPrintErrorTrace() totalT2 = float(cv2.getTickCount()) myTime = (totalT2 - totalT1) / cv2.getTickFrequency() print("QuadTreeDecision() took %.6f [sec]" % (myTime)) return crossref