def iteration_standalone_mqr(query_frame): r_quads_tree = g.r_quadsTree r_harlocs = g.r_harlocs q_harlocs = g.q_harlocs md_threshold = g.md_threshold st_threshold = g.st_threshold all_ori = g.all_ori all_id = g.all_id all_max = g.all_max all_cen = g.all_cen nos = g.nos scale_index = g.scale_index crop_flag = g.crop_flag sequence = g.sequence rd_start = g.rd_start rd_end = g.rd_end maxdis = g.maxdis maxori = g.maxori tolers = g.tolers """ We make pp reference the desired multiharloc list for the query video frame query_frame """ pp = q_harlocs[query_frame] """ Alex: for the query frame query_frame we retrieve, for scale scale_index, the harris features in var points. Then we build the quads from points. Then for each quad (4 float values) we query the corresponding scale kd-tree, and we get the indices. Then we build the histogram and compute idf, ....!!!! Note: scale is 1 for original frame resolution and the higher we go we have lower image resolutions (we go higher in the Guassian pyramid I think). """ points = pp[pp[:, 2] == scale_index, 0:2] qout, qcen, qmaxdis, qori = findquads.findquads(points, md_threshold, 0) common.DebugPrint("multiscale_quad_retrieval(): query_frame = %d, " "qout.shape = %s" % (query_frame, str(qout.shape))) space_xy = np.zeros((qcen.shape[0], 2 * len(r_harlocs))) + np.nan votes = np.zeros((len(r_harlocs), 1)) assert isinstance(tolers, float) """ We substitute queryFrameQuad - 1 with queryFrameQuad, since we want to number arrays from 0 (not from 1 like in Matlab). """ for queryFrameQuad in range(qout.shape[0]): """ Matlab's polymorphism is really bugging here: although it's normally a float, tolers is considered to be a size 1 vector... so len(tolers) == 1 """ """ We substitute tol_i - 1 with tol, since we want to number arrays from 0 (not from 1 like in Matlab). """ for tol_i in range(1): tol = tolers # default for first PAMI with tol= 0.1 approximately # NOTE: SciPy's KDTree finds a few more results, in some cases, # than the Matlab code from Evangelidis. # tol is a scalar representing the radius of the ball if config.KDTREE_IMPLEMENTATION == 0: idx = r_quads_tree.query_ball_point(qout[queryFrameQuad, :], tol) elif config.KDTREE_IMPLEMENTATION == 1: pt = qout[queryFrameQuad, :] pt = np.array([[pt[0], pt[1], pt[2], pt[3]]], dtype=np.float32) retval, idx, dists = r_quads_tree.radiusSearch( query=pt, radius=(tol ** 2), maxResults=NUM_MAX_ELEMS, params=search_params) if common.MY_DEBUG_STDOUT: common.DebugPrint( "multiscale_quad_retrieval(): radiusSearch's retval " "(at query_frame=%d, queryFrameQuad=%d) is %d\n" % ( query_frame, queryFrameQuad, retval)) idx = idx[0] dists = dists[0] idx = idx[: retval] dists = dists[: retval] if common.MY_DEBUG_STDOUT: print("multiscale_quad_retrieval(): " "qout[queryFrameQuad, :] = %s" % str( qout[queryFrameQuad, :])) print("multiscale_quad_retrieval(): " "idx = %s" % str(idx)) print("multiscale_quad_retrieval(): " "tol = %s" % str(tol)) if config.KDTREE_IMPLEMENTATION == 0: print("multiscale_quad_retrieval(): " "r_quads_tree.data[idx] = %s" % str(r_quads_tree.data[idx])) # We print the distances to the points returned in idx a = qout[queryFrameQuad, :] idx = np.array(idx) if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " "all_max.shape = %s" % str(all_max.shape)) common.DebugPrint("multiscale_quad_retrieval(): " "qmaxdis.shape = %s" % str(qmaxdis.shape)) common.DebugPrint("multiscale_quad_retrieval(): " "qmaxdis = %s" % str(qmaxdis)) common.DebugPrint("multiscale_quad_retrieval(): " "qori.shape = %s" % str(qori.shape)) common.DebugPrint("multiscale_quad_retrieval(): " "qori = %s" % str(qori)) if len(idx) == 0: # NOT A GOOD IDEA: continue # idx = np.array([]) dis_idx = np.array([]) ori_idx = np.array([]) else: if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " "queryFrameQuad = %s" % str( queryFrameQuad)) common.DebugPrint("multiscale_quad_retrieval(): " "all_max[idx] = %s" % str(all_max[idx])) common.DebugPrint("multiscale_quad_retrieval(): " "qmaxdis[queryFrameQuad] = %s" % str( qmaxdis[queryFrameQuad])) dis_idx = np.abs( qmaxdis[queryFrameQuad] - all_max[idx]) < maxdis if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " "dis_idx = %s" % str(dis_idx)) idx = idx[dis_idx] if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " "idx (after idx = idx[dis_idx]) = " "%s" % str(idx)) ori_idx = np.abs(qori[queryFrameQuad] - all_ori[idx]) < maxori if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " "ori_idx = %s" % str(ori_idx)) idx = idx[ori_idx] # IMPORTANT ################################################### # IMPORTANT ################################################### # IMPORTANT ################################################### # spatio-temporal consistency # IMPORTANT ################################################### # IMPORTANT ################################################### # IMPORTANT ################################################### if idx.size > 0: # Normally crop_flag == 0 if crop_flag == 0: dy = qcen[queryFrameQuad, 0] - all_cen[idx, 0] dx = qcen[queryFrameQuad, 1] - all_cen[idx, 1] D = dy ** 2 + dx ** 2 co_idx = D < pow(st_threshold, 2) idx = idx[co_idx] else: """ We substitute iii - 1 with iii, since we want to number arrays from 0 (not from 1 like in Matlab). """ for iii in range(len(idx)): space_xy[queryFrameQuad, (all_id[idx[iii]] - rd_start) * 2: (all_id[idx[ iii] - 1] - rd_start) * 2 + 1] = \ all_cen[idx[iii], :] # It has to be an np.array because we multiply it with a scalar histo_range = np.array(range(rd_start, rd_end + 1)) hh = Matlab.hist(x=all_id[idx], binCenters=histo_range) if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " "hh = %s" % (str(hh))) common.DebugPrint("multiscale_quad_retrieval(): " "hh.shape = %s" % (str(hh.shape))) common.DebugPrint("multiscale_quad_retrieval(): " "all_id.shape = %s" % (str(all_id.shape))) common.DebugPrint("multiscale_quad_retrieval(): " "idx = %s" % (str(idx))) common.DebugPrint("multiscale_quad_retrieval(): " "idx.shape = %s" % (str(idx.shape))) # nz can be computed more optimally # nz=find(hh~=0) # nz can be computed more optimally # np.nonzero() always returns a tuple, even if it contains # 1 element since hh has only 1 dimension nz = np.nonzero(hh != 0)[0] if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " "nz = %s" % (str(nz))) common.DebugPrint("multiscale_quad_retrieval(): " "nz.shape = %s" % (str(nz.shape))) if nz.size > 0: my_val = pow(math.log10(float(len(r_harlocs)) / len(nz)), 2) if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " "len(r_harlocs) = " "%d" % len(r_harlocs)) common.DebugPrint("multiscale_quad_retrieval(): " "len(nz) = %d" % len(nz)) common.DebugPrint("multiscale_quad_retrieval(): " "my_val = %.5f" % my_val) # PREVIOUSLY votes[nz, tol_i] = votes[nz, tol_i] + my_val if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " "votes.shape = %s" % (str(votes.shape))) common.DebugPrint("multiscale_quad_retrieval(): " "votes = %s" % (str(votes))) return query_frame, np.ravel(votes)
def my_nms(cim, radius, thresh): common.DebugPrint("Entered my_nms(cim.shape=%s, radius=%s, thresh=%s)" % \ (str(cim.shape), str(radius), str(thresh))) #%% modification of Peter-Kovesi non-maximum suppression software by #%% G.Evangelidis common.DebugPrint("my_nms(): cim = %s" % str(cim)) #%subPixel = nargout == 4; % We want sub-pixel locations #subPixel=1; subPixel = 1 #[rows,cols] = size(cim) rows, cols = cim.shape common.DebugPrint("my_nms(): rows, cols (cim.shape) = %s" % str( (rows, cols))) #% Extract local maxima by performing a grey scale morphological #% dilation and then finding points in the corner strength image that #% match the dilated image and are also greater than the threshold. sze = 2 * radius + 1 #% Size of dilation mask. common.DebugPrint("my_nms(): cim.shape = %s" % str(cim.shape)) #common.DebugPrint("my_nms(): cim = %s" % str(cim)); common.DebugPrint("my_nms(): sze = %s" % str(sze)) """ Alex: we pass sze only being odd number and order = sze^2 makes ordfilt2() return the maximum from the domain. """ #%% This modification runs 4x faster (11 secs less in a 6-scale approach) #mx = ordfilt2(cim,sze^2,ones(sze)); #% Grey-scale dilate. mx = Matlab.ordfilt2(cim, pow(sze, 2), np.ones((sze, sze))) #% Grey-scale dilate. if common.MY_DEBUG_STDOUT: common.DebugPrint("my_nms(): mx = %s" % str(mx)) #% mx=my_max_filter(cim,[sze,sze]); %my mex-file for max-filter #%% This modification verify that the central point is the unique maximum in #%% neighborhood #% mx2= ordfilt2(cim,sze^2-1,ones(sze)); % This is used to vrify that the #% central point is the unique maximum in neighborhood #% imagesc(cim) #% pause #% imagesc(mx) #% pause #% close #% Make mask to exclude points within radius of the image boundary. #bordermask = zeros(size(cim)); #bordermask = np.zeros(cim.shape); bordermask = np.zeros(cim.shape, dtype=np.uint8) #Alex: sets to 1 the matrix, except the first and last radius lines and first and last radius columns, which are left 0 #bordermask(radius+1:end-radius, radius+1:end-radius) = 1; bordermask[radius:-radius, radius:-radius] = 1 #% Find maxima, threshold, and apply bordermask #cimmx = (cim==mx) & (cim>thresh) & bordermask; #%Peter-Kovesi cimmx = np.logical_and( \ np.logical_and((cim == mx), (cim > thresh)), \ bordermask) #%Peter-Kovesi #% cimmx = (cim==mx) & (cim~=mx2) & (cim>thresh) & bordermask; %my modification common.DebugPrint("my_nms(): cimmx.shape = %s" % str(cimmx.shape)) #common.DebugPrint("my_nms(): cimmx = %s" % str(cimmx)); #[r,c] = find(cimmx) #% Find row,col coords. if True: """ Retrieving indices in "Fortran" (column-major) order, like in Matlab. We do this in order to get the harris points sorted like in Matlab. """ c, r = np.nonzero(cimmx.T) else: # Retrieving indices in row-major order. r, c = np.nonzero(cimmx) common.DebugPrint("my_nms(): r = %s" % str(r)) common.DebugPrint("my_nms(): c = %s" % str(c)) if subPixel: #% Compute local maxima to sub pixel accuracy # From Matlab help: "Determine whether array is empty" "An empty array has at least one dimension of size zero, for example, 0-by-0 or 0-by-5." #if not isempty(r): #% ...if we have some ponts to work with if r.size > 0: #% ...if we have some ponts to work with #ind = sub2ind(size(cim),r,c); #% 1D indices of feature points ind = Matlab.sub2ind(cim.shape, r, c) #% 1D indices of feature points w = 1 #% Width that we look out on each side of the feature #% point to fit a local parabola if common.MY_DEBUG_STDOUT: common.DebugPrint("my_nms(): ind.shape = %s" % str(ind.shape)) common.DebugPrint("my_nms(): ind = %s" % str(ind)) common.DebugPrint("my_nms(): ind - w = %s" % str(ind - w)) # Don't forget that we are now in column major ('F'/Fortran) order #% Indices of points above, below, left and right of feature point #indrminus1 = max(ind-w,1); #indrminus1 = np.max(ind - w, 0); # In Matlab this is what it returns for 1D ind: assert ind.ndim == 1 indrminus1 = ind - w #indrplus1 = min(ind+w,rows*cols); #indrplus1 = np.min(ind + w, rows * cols); # In Matlab this is what it returns for 1D ind: assert ind.ndim == 1 indrplus1 = ind + w #indcminus1 = max(ind-w*rows,1); #indcminus1 = np.max(ind - w * rows, 1); # In Matlab this is what it returns for 1D ind: assert ind.ndim == 1 indcminus1 = ind - w * rows #indcplus1 = min(ind+w*rows,rows*cols); #indcplus1 = np.min(ind + w * rows, rows * cols); # In Matlab this is what it returns for 1D ind: assert ind.ndim == 1 indcplus1 = ind + w * rows # De-flattening the index arrays back into tuple, as accepted by numpy # See http://docs.scipy.org/doc/numpy/reference/generated/numpy.unravel_index.html ind = np.unravel_index(ind, cim.shape, order="F") indrminus1 = np.unravel_index(indrminus1, cim.shape, order="F") indrplus1 = np.unravel_index(indrplus1, cim.shape, order="F") indcminus1 = np.unravel_index(indcminus1, cim.shape, order="F") indcplus1 = np.unravel_index(indcplus1, cim.shape, order="F") #% Solve for quadratic down rows #cy = cim(ind); cy = cim[ind] # In Matlab ay has float elements ay = (cim[indrminus1] + cim[indrplus1]) / 2.0 - cy #by = ay + cy - cim(indrminus1); by = ay + cy - cim[indrminus1] #rowshift = -w*by./(2*ay); #% Maxima of quadradic rowshift = -w * by / (2.0 * ay) #% Maxima of quadradic #% Solve for quadratic across columns #cx = cim(ind); cx = cim[ind] #ax = (cim(indcminus1) + cim(indcplus1))/2 - cx; ax = (cim[indcminus1] + cim[indcplus1]) / 2.0 - cx #bx = ax + cx - cim(indcminus1); bx = ax + cx - cim[indcminus1] #colshift = -w*bx./(2*ax); #% Maxima of quadratic colshift = -w * bx / (2.0 * ax) #% Maxima of quadratic rsubp = r + rowshift #% Add subpixel corrections to original row csubp = c + colshift #% and column coords. else: #rsubp = []; csubp = []; rsubp = np.array([]) csubp = np.array([]) """ % if nargin==4 & ~isempty(r) % Overlay corners on supplied image. % figure, imshow(im,[]), hold on % if subPixel % plot(csubp,rsubp,'r+'), title('corners detected'); % else % plot(c,r,'r+'), title('corners detected'); % end % end """ return r, c, rsubp, csubp
def IterationStandaloneMQR(queryFrame): r_quadsTree = g.r_quadsTree r_harlocs = g.r_harlocs q_harlocs = g.q_harlocs md_threshold = g.md_threshold st_threshold = g.st_threshold all_ori = g.all_ori all_id = g.all_id all_max = g.all_max all_cen = g.all_cen nos = g.nos scale_index = g.scale_index cropflag = g.cropflag sequence = g.sequence RD_start = g.RD_start RD_end = g.RD_end MAXDIS = g.MAXDIS MAXORI = g.MAXORI tolers = g.tolers """ common.DebugPrint( \ "Entered IterationStandaloneMQR(): crossref=%s, captureQ=%s, "\ "captureR=%s, refined_crossref=%s, warp_p=%s, " "x0=%s, y0=%s, start=%s, t=%d, iWhile=%d." % \ (str(crossref), str(captureQ), str(captureR), \ str(g.refined_crossref), str(g.warp_p), \ str(g.x0), str(g.y0), str(g.start), g.t, iWhile)); common.DebugPrint("IterationStandalone(): id(g)=%s" % str(id(g))); """ # tic """ str1=['load ' q_path QD(q).name] eval(str1) """ """ We make pp reference the desired multiharloc list for the query video frame queryFrame """ pp = q_harlocs[queryFrame] #pp = np.array(pp); #common.DebugPrint("multiscale_quad_retrieval(): pp = %s" % str(pp)); """ Alex: for the query frame queryFrame we retrieve, for scale scale_index, the harris features in var points. Then we build the quads from points. Then for each quad (4 float values) we query the corresponding scale kd-tree, and we get the indices. Then we build the histogram and compute idf, ....!!!! Note: scale is 1 for original frame resolution and the higher we go we have lower image resolutions (we go higher in the Guassian pyramid I think). """ #[qout,qcen,qmaxdis,qori]=findquads(pp(pp(:,3)==scale_index,1:2),md_threshold,0); points = pp[pp[:, 2] == scale_index, 0:2] qout, qcen, qmaxdis, qori = findquads.findquads(points, md_threshold, 0) common.DebugPrint("multiscale_quad_retrieval(): queryFrame = %d, " \ "qout.shape = %s" % (queryFrame, str(qout.shape))) # disp([num2str(q) ' of ' num2str(length(QD)) ' -> ' num2str(size(qout,1)) ' quads']) #space_xy=zeros(size(qcen,1),2*length(RD))+nan; #space_xy = np.zeros( (qcen.shape[0], 2 * len(RD)) ) + np.nan; space_xy = np.zeros((qcen.shape[0], 2 * len(r_harlocs))) + np.nan # votes=zeros(length(RD),1) #votes=zeros(length(RD),length(tolers)); #votes = np.zeros( (len(RD), 1) ); votes = np.zeros((len(r_harlocs), 1)) #nep = np.array([]); #m_points = np.array([]); assert isinstance(tolers, float) """ We substitute queryFrameQuad - 1 with queryFrameQuad, since we want to number arrays from 0 (not from 1 like in Matlab). """ #for queryFrameQuad in range(1, qout.shape[0] + 1): for queryFrameQuad in range(qout.shape[0]): """ Matlab's polymorphism is really bugging here: although it's normally a float, tolers is considered to be a size 1 vector... so len(tolers) == 1 """ #for tol_i in range(1, len(tolers) + 1): # tol = tolers[tol_i - 1] """ We substitute tol_i - 1 with tol, since we want to number arrays from 0 (not from 1 like in Matlab). """ #for tol_i in range(1, 1 + 1): for tol_i in range(1): tol = tolers #common.DebugPrint("multiscale_quad_retrieval(): qout[i - 1, :] = %s" % str(qout[i - 1, :])) #% default for first PAMI with tol= 0.1 approximately # NOTE: SciPy's KDTree finds a few more results, in some cases, # than the Matlab code from Evangelidis. #idx, di = kdtree_ball_query(tree, qout(i, :), tol) #idx, distKD = kdtree_ball_query(tree, qout[i - 1, :], tol) #idx, di = tree.query(x=xQuery, k=4) #resPoints = [data[i] for i in resBallIndices] # tol is a scalar representing the radius of the ball if config.KDTREE_IMPLEMENTATION == 0: idx = r_quadsTree.query_ball_point(qout[queryFrameQuad, :], tol) elif config.KDTREE_IMPLEMENTATION == 1: #pt = qout[queryFrameQuad - 1, :].astype(np.float32); pt = qout[queryFrameQuad, :] pt = np.array([[pt[0], pt[1], pt[2], pt[3]]], dtype=np.float32) retval, idx, dists = r_quadsTree.radiusSearch( \ query=pt, \ radius=(tol**2), \ maxResults=NUM_MAX_ELEMS, \ params=search_params) if common.MY_DEBUG_STDOUT and DBGPRINT: """ common.DebugPrint("multiscale_quad_retrieval(): " \ "retval (number NNs) = %s" % str(retval)); """ common.DebugPrint( \ "multiscale_quad_retrieval(): radiusSearch's retval " \ "(at queryFrame=%d, queryFrameQuad=%d) is %d\n" % (queryFrame, queryFrameQuad, retval)) idx = idx[0] dists = dists[0] idx = idx[:retval] dists = dists[:retval] if common.MY_DEBUG_STDOUT and DBGPRINT: print("multiscale_quad_retrieval(): " \ "qout[queryFrameQuad, :] = %s" % str(qout[queryFrameQuad, :])) print("multiscale_quad_retrieval(): " \ "idx = %s" % str(idx)) print("multiscale_quad_retrieval(): " \ "tol = %s" % str(tol)) if config.KDTREE_IMPLEMENTATION == 0: print("multiscale_quad_retrieval(): " \ "r_quadsTree.data[idx] = %s" % \ str(r_quadsTree.data[idx])) # We print the distances to the points returned in idx a = qout[queryFrameQuad, :] if False: #!!!! This is just for debugging purposes for myI, index in enumerate(idx): b = r_quadsTree.data[index] """ if False: common.DebugPrint("multiscale_quad_retrieval(): distance to " \ "%d point (%s) inside ball = %.4f" % \ (myI, str(b), npla.norm(a - b))); """ idx = np.array(idx) #if False: if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " \ "all_max.shape = %s" % str(all_max.shape)) common.DebugPrint("multiscale_quad_retrieval(): " \ "qmaxdis.shape = %s" % str(qmaxdis.shape)) common.DebugPrint("multiscale_quad_retrieval(): " \ "qmaxdis = %s" % str(qmaxdis)) common.DebugPrint("multiscale_quad_retrieval(): " \ "qori.shape = %s" % str(qori.shape)) common.DebugPrint("multiscale_quad_retrieval(): " \ "qori = %s" % str(qori)) #dis_idx=abs(qmaxdis(i)-all_max(idx))<MAXDIS; if len(idx) == 0: # NOT A GOOD IDEA: continue; #idx = np.array([]); dis_idx = np.array([]) ori_idx = np.array([]) else: #if False: if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " \ "queryFrameQuad = %s" % str(queryFrameQuad)) common.DebugPrint("multiscale_quad_retrieval(): " \ "all_max[idx] = %s" % str(all_max[idx])) common.DebugPrint("multiscale_quad_retrieval(): " \ "qmaxdis[queryFrameQuad] = %s" % str(qmaxdis[queryFrameQuad])) dis_idx = np.abs(qmaxdis[queryFrameQuad] - all_max[idx]) < MAXDIS #if False: if common.MY_DEBUG_STDOUT: """ common.DebugPrint("multiscale_quad_retrieval(): " \ "idx = %s" % str(idx)); """ common.DebugPrint("multiscale_quad_retrieval(): " \ "dis_idx = %s" % str(dis_idx)) #idx=idx(dis_idx) idx = idx[dis_idx] #if False: if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " \ "idx (after idx = idx[dis_idx]) = %s" % str(idx)) #ori_idx=abs(qori(i)-all_ori(idx))<MAXORI; ori_idx = np.abs(qori[queryFrameQuad] - all_ori[idx]) < MAXORI #if False: if common.MY_DEBUG_STDOUT: """ common.DebugPrint("multiscale_quad_retrieval(): " \ "all_ori = %s" % str(all_ori)); common.DebugPrint("multiscale_quad_retrieval(): " \ "qori[queryFrameQuad] = %s" % str(qori[queryFrameQuad])); """ common.DebugPrint("multiscale_quad_retrieval(): " \ "ori_idx = %s" % str(ori_idx)) #idx=idx(ori_idx); idx = idx[ori_idx] # IMPORTANT ################################################### # IMPORTANT ################################################### # IMPORTANT ################################################### #% spatio-temporal consistency # IMPORTANT ################################################### # IMPORTANT ################################################### # IMPORTANT ################################################### #if numel(idx) > 0: if idx.size > 0: # Normally cropflag == 0 if cropflag == 0: dy = qcen[queryFrameQuad, 0] - all_cen[idx, 0] dx = qcen[queryFrameQuad, 1] - all_cen[idx, 1] #D=dy.^2+dx.^2; D = dy**2 + dx**2 co_idx = D < pow(st_threshold, 2) idx = idx[co_idx] else: """ We substitute iii - 1 with iii, since we want to number arrays from 0 (not from 1 like in Matlab). """ #for iii in range(1, len(idx) + 1): for iii in range(len(idx)): #space_xy(i,(all_id(idx(iii))-RD_start)*2+1:(all_id(idx(iii))-RD_start)*2+2) = all_cen(idx(iii),:) space_xy[queryFrameQuad, \ (all_id[idx[iii]] - RD_start) * 2: (all_id[idx[iii] - 1] - RD_start) * 2 + 1] = \ all_cen[idx[iii], :] #hh=hist(all_id(idx),RD_start:RD_end); # It has to be an np.array because we multiply it with a scalar histoRange = np.array(range(RD_start, RD_end + 1)) hh = Matlab.hist(x=all_id[idx], binCenters=histoRange) #if False: if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " \ "hh = %s" % (str(hh))) common.DebugPrint("multiscale_quad_retrieval(): " \ "hh.shape = %s" % (str(hh.shape))) """ common.DebugPrint("multiscale_quad_retrieval(): " \ "all_id = %s" % (str(all_id))); """ common.DebugPrint("multiscale_quad_retrieval(): " \ "all_id.shape = %s" % (str(all_id.shape))) common.DebugPrint("multiscale_quad_retrieval(): " \ "idx = %s" % (str(idx))) common.DebugPrint("multiscale_quad_retrieval(): " \ "idx.shape = %s" % (str(idx.shape))) # % nz can be computed more optimally #nz=find(hh~=0); # nz can be computed more optimally # np.nonzero() always returns a tuple, even if it contains 1 element since hh has only 1 dimension nz = np.nonzero(hh != 0)[0] #if False: if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " \ "nz = %s" % (str(nz))) common.DebugPrint("multiscale_quad_retrieval(): " \ "nz.shape = %s" % (str(nz.shape))) #if numel(nz) > 0: if nz.size > 0: #%%----text-retrieval-like #votes(nz, tol_i) = votes(nz, tol_i) + log10(length(RD) / (length(nz)))^2 #Note: log10(a)^2 means (log10(a))^2 #PREVIOUSLY #myVal = pow(math.log10(float(len(RD)) / len(nz)), 2); myVal = pow(math.log10(float(len(r_harlocs)) / len(nz)), 2) #if False: if common.MY_DEBUG_STDOUT: """ common.DebugPrint("multiscale_quad_retrieval(): " \ "len(RD) = %d" % len(RD)); """ common.DebugPrint("multiscale_quad_retrieval(): " \ "len(r_harlocs) = %d" % len(r_harlocs)) common.DebugPrint("multiscale_quad_retrieval(): " \ "len(nz) = %d" % len(nz)) common.DebugPrint("multiscale_quad_retrieval(): " \ "myVal = %.5f" % myVal) # PREVIOUSLY votes[nz, tol_i] = votes[nz, tol_i] + myVal # votes(nz)=votes(nz)+log10(length(RD)/(length(nz))); # votes(nz)=votes(nz)+1; #if False: if common.MY_DEBUG_STDOUT: """ common.DebugPrint("multiscale_quad_retrieval(): " \ "Votes_space.shape = %s" % (str(Votes_space.shape))); common.DebugPrint("multiscale_quad_retrieval(): " \ "votes.shape = %s" % (str(votes.shape))); """ common.DebugPrint("multiscale_quad_retrieval(): " \ "votes.shape = %s" % (str(votes.shape))) common.DebugPrint("multiscale_quad_retrieval(): " \ "votes = %s" % (str(votes))) return (queryFrame, np.ravel(votes)) # NOT performing these in each worker - the central dispatcher will do these if False: #Votes_space(:,q)=votes; # Gives: "ValueError: output operand requires a reduction, but reduction is not enabled" #Votes_space[:, queryFrame - 1] = votes; Votes_space[:, queryFrame] = np.ravel(votes) if cropflag == 0: HH[:, queryFrame] = 1 else: """ HH[:, queryFrame] = spatial_consistency.spatial_consistency(space_xy, \ qcen, len(RD), st_threshold, cropflag); """ HH[:, queryFrame] = spatial_consistency.spatial_consistency(space_xy, \ qcen, len(r_harlocs), st_threshold, cropflag)
def multiscale_quad_retrieval(r_quads_tree, r_harlocs, q_harlocs, md_threshold, st_threshold, all_ori, all_id, all_max, all_cen, nos, scale_index, crop_flag, sequence): common.DebugPrint("Entered multiscale_quad_retrieval(): " "md_threshold = %s, st_threshold = %s." % (str(md_threshold), str(st_threshold))) assert len(r_harlocs) != 0 assert len(q_harlocs) != 0 try: votes_space = np.load("votes_space%d.npz" % scale_index)['arr_0'] HH = np.load("HH%d.npz" % scale_index)['arr_0'] return votes_space, HH except: common.DebugPrintErrorTrace() if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): r_quads_tree = %s" % str(r_quads_tree)) common.DebugPrint( "multiscale_quad_retrieval(): len(r_harlocs) = %d" % len(r_harlocs)) common.DebugPrint( "multiscale_quad_retrieval(): r_harlocs = %s" % str(r_harlocs)) common.DebugPrint( "multiscale_quad_retrieval(): q_harlocs = %s" % str(q_harlocs)) common.DebugPrint( "multiscale_quad_retrieval(): md_threshold = %s" % str( md_threshold)) print("multiscale_quad_retrieval(): st_threshold = %s" % str( st_threshold)) common.DebugPrint( "multiscale_quad_retrieval(): all_id = %s" % str(all_id)) common.DebugPrint("multiscale_quad_retrieval(): all_id.shape = %s" % ( str(all_id.shape))) common.DebugPrint( "multiscale_quad_retrieval(): sequence = %s" % str(sequence)) print("multiscale_quad_retrieval(): crop_flag = %s" % str(crop_flag)) t1 = float(cv2.getTickCount()) if scale_index > nos: assert scale_index <= nos # TODO: take out rd_start rd_start = 0 rd_end = len(r_harlocs) - 1 j = 1 """ Inspired from https://stackoverflow.com/questions/17559140/matlab-twice-as-fast-as-numpy BUT doesn't help in this case: votes_space = np.asfortranarray(np.zeros( (len(RD), len(QD)) )) """ votes_space = np.zeros((len(r_harlocs), len(q_harlocs))) # Make a distinct copy of HH from votes_space... # TODO: use MAYBE even np.bool - OR take it out HH = np.zeros((len(r_harlocs), len(q_harlocs)), dtype=np.int8) # it helps to make more strict the threshold as the scale goes up tolers = 0.1 - float(scale_index) / 100.0 maxdis = 3 + scale_index maxori = 0.25 # TODO: I am using multiprocessing.Poll and return votes the dispatcher # assembles the results, but the results are NOT the same with the serial # case - although they look pretty decent, but they seem to be # suboptimal - dp_alex returns suboptimal cost path for # USE_MULTITHREADING == True instead of False. # (Note: running under the same preconditions # multiscale_quad_retrieval I got the same results in dp_alex(). """ if False: #config.USE_MULTITHREADING == True: global g g.r_quads_tree = r_quads_tree g.r_harlocs = r_harlocs g.q_harlocs = q_harlocs g.md_threshold = md_threshold g.st_threshold = st_threshold g.all_ori = all_ori g.all_id = all_id g.all_max = all_max g.all_cen = all_cen g.nos = nos g.scale_index = scale_index g.crop_flag = crop_flag g.sequence = sequence g.RD_start = RD_start g.RD_end = RD_end g.maxdis = maxdis g.maxori = maxori g.tolers = tolers #Start worker processes to use on multi-core processor (able to run # in parallel - no GIL issue if each core has it's own VM) pool = multiprocessing.Pool(processes=config.numProcesses) print("multiscale_quad_retrieval(): Spawned a pool of %d workers" % config.numProcesses) listParams = range(0, len(q_harlocs)) #!!!!TODO: use counterStep, config.initFrame[indexVideo] #res = pool.map(iteration_standalone_mqr, listParams) # See https://docs.python.org/2/library/multiprocessing.html#module-multiprocessing.pool res = pool.map(func=iteration_standalone_mqr, iterable=listParams, chunksize=1) print("Pool.map returns %s" % str(res)) #x0.size + 1 # From https://medium.com/building-things-on-the-internet/40e9b2b36148 # close the pool and wait for the work to finish pool.close() pool.join() # Doing the "reduce" phase after the workers have finished :) assert len(res) == len(q_harlocs) for query_frame, resE in enumerate(res): resEIndex = resE[0] resE = resE[1] assert resEIndex == query_frame # Gives: "ValueError: output operand requires a reduction, but reduction is not enabled" #votes_space[:, query_frame - 1] = votes votes_space[:, query_frame] = resE for query_frame in range(len(q_harlocs)): if crop_flag == 0: HH[:, query_frame] = 1 else: HH[:, query_frame] = spatial_consistency.spatial_consistency(space_xy, qcen, len(r_harlocs), st_threshold, crop_flag) try: np.savez_compressed("votes_space%d" % scale_index, votes_space) np.savez_compressed("HH%d" % scale_index, HH) except: common.DebugPrintErrorTrace() return votes_space, HH """ """ We substitute q - 1 with q, since we want to number arrays from 0 (not from 1 like in Matlab). """ for query_frame in range(len(q_harlocs)): common.DebugPrint("multiscale_quad_retrieval(): Starting iteration " "query_frame = %d" % query_frame) """ We make pp reference the desired multiharloc list for the query video frame query_frame """ pp = q_harlocs[query_frame] points = pp[pp[:, 2] == scale_index, 0:2] qout, qcen, qmaxdis, qori = findquads.findquads(points, md_threshold, 0) if common.MY_DEBUG_STDOUT: print("multiscale_quad_retrieval(): query_frame = %d, " "qout.shape (number of quads for query frame query_frame) = " "%s" % (query_frame, str(qout.shape))) space_xy = np.zeros((qcen.shape[0], 2 * len(r_harlocs))) + np.nan votes = np.zeros((len(r_harlocs), 1)) assert isinstance(tolers, float) if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): quads of query " "frame %d are: " % query_frame) common.DebugPrint(" qout = %s" % str(qout)) """ Alex: for each quad (4 floats) of the query frame from Harris feature of scale scale_index Note: all_id stores the reference frame id for each quad descriptor. """ """ We substitute queryFrameQuad - 1 with queryFrameQuad, since we want to number arrays from 0 (not from 1 like in Matlab). """ for queryFrameQuad in range(qout.shape[0]): common.DebugPrint("multiscale_quad_retrieval(): Starting iteration " "queryFrameQuad = %d" % queryFrameQuad) """ Matlab's polymorphism is really bugging here: although it's normally a float, tolers is considered to be a size 1 vector... so len(tolers) == 1 """ """ We substitute tol_i - 1 with tol, since we want to number arrays from 0 (not from 1 like in Matlab). """ for tol_i in range(1): tol = tolers # default for first PAMI with tol= 0.1 approximately # NOTE: SciPy's KDTree finds a few more results, in some cases, # than the Matlab code from Evangelidis. # tol is a scalar representing the radius of the ball if config.KDTREE_IMPLEMENTATION == 0: idx = r_quads_tree.query_ball_point(qout[queryFrameQuad, :], tol) elif config.KDTREE_IMPLEMENTATION == 1: pt = qout[queryFrameQuad, :] pt = np.array([[pt[0], pt[1], pt[2], pt[3]]], dtype=np.float32) retval, idx, dists = r_quads_tree.radiusSearch( query=pt, radius=(tol ** 2), maxResults=NUM_MAX_ELEMS, params=search_params) if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " "radiusSearch's retval (at " "query_frame=%d, queryFrameQuad=%d) " "is %d" % (query_frame, queryFrameQuad, retval)) idx = idx[0] dists = dists[0] """ Note: retval is the number of neighbors returned from the radiusSearch(). But the idx and the dists can have more elements than the returned retval. """ idx = idx[: retval] dists = dists[: retval] if common.MY_DEBUG_STDOUT: print("multiscale_quad_retrieval(): " "qout[queryFrameQuad, :] = %s" % str(qout[queryFrameQuad, :])) print("multiscale_quad_retrieval(): " "idx = %s" % str(idx)) print("multiscale_quad_retrieval(): " "dists = %s" % str(dists)) print("multiscale_quad_retrieval(): " "tol = %s" % str(tol)) if config.KDTREE_IMPLEMENTATION == 0: print("multiscale_quad_retrieval(): " "r_quads_tree.data[idx] = %s" % str(r_quads_tree.data[idx])) if common.MY_DEBUG_STDOUT: a = qout[queryFrameQuad, :] if config.KDTREE_IMPLEMENTATION == 0: for myI, index in enumerate(idx): b = r_quads_tree.data[index] else: pass idx = np.array(idx) if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " "all_max.shape = %s" % str(all_max.shape)) common.DebugPrint("multiscale_quad_retrieval(): " "qmaxdis.shape = %s" % str(qmaxdis.shape)) common.DebugPrint("multiscale_quad_retrieval(): " "qmaxdis = %s" % str(qmaxdis)) common.DebugPrint("multiscale_quad_retrieval(): " "qori.shape = %s" % str(qori.shape)) common.DebugPrint("multiscale_quad_retrieval(): " "qori = %s" % str(qori)) if len(idx) == 0: # NOT A GOOD IDEA: continue dis_idx = np.array([]) ori_idx = np.array([]) else: if common.MY_DEBUG_STDOUT: print("multiscale_quad_retrieval(): " "queryFrameQuad = %s" % str(queryFrameQuad)) print("multiscale_quad_retrieval(): " "all_max[idx] = %s" % str(all_max[idx])) print("multiscale_quad_retrieval(): " "qmaxdis[queryFrameQuad] = %s" % str(qmaxdis[queryFrameQuad])) if USE_GPS_COORDINATES: # We look only at a part of the reference video """ Since in some cases the video temporal alignment is difficult to do due to similar portions in the trajectory (see the drone videos, clip 3_some_lake) we "guide" the temporal alignment by restricting the reference frame search space - this is useful when we have the geolocation (GPS) coordinate for each frame. """ if common.MY_DEBUG_STDOUT: print("multiscale_quad_retrieval(): " "all_id = %s" % str(all_id)) if all_id.ndim == 2: # TODO: put this at the beginning of the # function assert all_id.shape[1] == 1 """ We flatten the array all_id Note: We don't use order="F" since it's basically 1-D array """ all_id = np.ravel(all_id) # TODO: put start and end frame in config - or compute # it from geolocation sub_idx = np.logical_and((all_id[idx] >= 2030 - 928), (all_id[idx] <= 2400 - 928)) idx = idx[sub_idx] if common.MY_DEBUG_STDOUT: print("multiscale_quad_retrieval(): " "all_id = %s" % str(all_id)) print("multiscale_quad_retrieval(): " "sub_idx = %s" % str(sub_idx)) print("multiscale_quad_retrieval(): " "idx = %s" % str(idx)) if FILTER: dis_idx = np.abs( qmaxdis[queryFrameQuad] - all_max[idx]) < maxdis if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " "dis_idx = %s" % str(dis_idx)) idx = idx[dis_idx] if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " "idx (after idx = idx[dis_idx]) = " "%s" % str(idx)) if FILTER: ori_idx = np.abs( qori[queryFrameQuad] - all_ori[idx]) < maxori if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " "ori_idx = %s" % str(ori_idx)) idx = idx[ori_idx] # IMPORTANT ################################################### # IMPORTANT ################################################### # IMPORTANT ################################################### # spatio-temporal consistency # IMPORTANT ################################################### # IMPORTANT ################################################### # IMPORTANT ################################################### if idx.size > 0: if crop_flag == 0: if FILTER: """ Alex: this is a simple procedure of eliminating False Positive (FP) matches, as presented in Section 4.2 of TPAMI 2013 paper. Basically it filters out quad matches that have centroids st_threshold away from the query quad. Note: all_cen are the controids of all reference quads. """ dy = qcen[queryFrameQuad, 0] - all_cen[idx, 0] dx = qcen[queryFrameQuad, 1] - all_cen[idx, 1] D = dy ** 2 + dx ** 2 co_idx = D < pow(st_threshold, 2) idx = idx[co_idx] else: """ We substitute iii - 1 with iii, since we want to number arrays from 0 (not from 1 like in Matlab). """ for iii in range(len(idx)): space_xy[queryFrameQuad, (all_id[idx[iii]] - rd_start) * 2: (all_id[idx[ iii] - 1] - rd_start) * 2 + 1] = \ all_cen[idx[iii], :] # It has to be an np.array because we multiply it with a # scalar histo_range = np.array(range(rd_start, rd_end + 1)) hh = Matlab.hist(x=all_id[idx], binCenters=histo_range) if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " "hh = %s" % (str(hh))) common.DebugPrint("multiscale_quad_retrieval(): " "hh.shape = %s" % (str(hh.shape))) common.DebugPrint("multiscale_quad_retrieval(): " "all_id.shape = %s" % ( str(all_id.shape))) common.DebugPrint("multiscale_quad_retrieval(): " "idx = %s" % (str(idx))) common.DebugPrint("multiscale_quad_retrieval(): " "idx.shape = %s" % (str(idx.shape))) # % nz can be computed more optimally nz = np.nonzero(hh != 0)[0] if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " "nz = %s" % (str(nz))) common.DebugPrint("multiscale_quad_retrieval(): " "nz.shape = %s" % (str(nz.shape))) if nz.size > 0: my_val = pow( math.log10(float(len(r_harlocs)) / len(nz)), 2) if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " "len(r_harlocs) = %d" % len( r_harlocs)) common.DebugPrint("multiscale_quad_retrieval(): " "len(nz) = %d" % len(nz)) common.DebugPrint("multiscale_quad_retrieval(): " "my_val = %.5f" % my_val) # PREVIOUSLY votes[nz, tol_i] = votes[nz, tol_i] + my_val if common.MY_DEBUG_STDOUT: print("multiscale_quad_retrieval(): " "votes.shape = %s" % (str(votes.shape))) if (np.abs(votes) < 1.0e-10).all(): print("multiscale_quad_retrieval(): votes = 0 (all zeros)") else: print("multiscale_quad_retrieval(): votes = %s" % (str(votes))) # Note: since votes is basically a 1-D vector, we don't use the # Fortran order votes_space[:, query_frame] = np.ravel(votes) if crop_flag == 0: HH[:, query_frame] = 1 else: HH[:, query_frame] = spatial_consistency.spatial_consistency( space_xy, qcen, len(r_harlocs), st_threshold, crop_flag) if common.MY_DEBUG_STDOUT: print("multiscale_quad_retrieval(scale_index=%d): " "votes_space =\n%s" % (scale_index, str(votes_space))) try: np.savez_compressed("votes_space%d" % scale_index, votes_space) np.savez_compressed("HH%d" % scale_index, HH) except: common.DebugPrintErrorTrace() t2 = float(cv2.getTickCount()) my_time = (t2 - t1) / cv2.getTickFrequency() print("multiscale_quad_retrieval() took %.6f [sec]" % my_time) return votes_space, HH
def multi_scale_harris_Evangelidis(im, nos, disp): common.DebugPrint("Entered multi_scale_harris_Evangelidis(nos=%d, disp=%s)" % \ (nos, str(disp))); #tic #if size(im, 3) == 3: # im = rgb2gray(im) tMSH1 = float(cv2.getTickCount()); if im.ndim == 3: im = common.ConvertImgToGrayscale(im); #im = im2double(im) # From https://stackoverflow.com/questions/10873824/how-to-convert-2d-float-numpy-array-to-2d-int-numpy-array: #DO NOT USE: im = im.astype(np.uint8); # This messes up tremendously the computation of harlocs #im = im.astype(float); im = im.astype(np.float32); #!!!!COR is an UNused variable #COR = zeros(size(im,1), size(im,2), size(im,3)) #COR = np.zeros((im.shape[0], im.shape[1], im.shape[1])); # scale values sigma_0 = 1.2; #n=[0:nos-1]; %scale levels n = np.array(range(nos)); #scale levels #sigma_D=sqrt(1.8).^n*sigma_0 sigma_D = math.sqrt(1.8) ** n * sigma_0; #points=[]; points = np.array([]); #scn=sigma_D.^4; scn = sigma_D ** 4; #for i=1:length(sigma_D) #for i in range(1, len(sigma_D) + 1): for i in range(1, nos + 1): common.DebugPrint("multi_scale_harris_Evangelidis(): i = %s" % (str(i))); #sd=sigma_D(i); %differentiation (local) scale sd = sigma_D[i - 1]; #%differentiation (local) scale #si=sd/.5; %integration scale si = sd / 0.5; #integration scale w = 3 * sd; # size for gaussian kernel to compute derivatives: 3 times local_scale r_w = int(round(w)); common.DebugPrint("multi_scale_harris_Evangelidis(): r_w = %s" % \ (str(r_w))); #if mod(round(w),2): if (r_w % 2) == 1: #[xco,yco] = meshgrid(-round(w):round(w),-round(w):round(w)) mRange = range(-r_w, r_w + 1); xco, yco = Matlab.meshgrid(mRange, mRange); else: #[xco,yco] = meshgrid(-(round(w)+1):round(w)+1,-(round(w)+1):round(w)+1) mRange = range(-(r_w + 1), r_w + 2); xco, yco = Matlab.meshgrid(mRange, mRange); if common.MY_DEBUG_STDOUT: common.DebugPrint("multi_scale_harris_Evangelidis(): xco = %s" % \ str(xco)); common.DebugPrint("multi_scale_harris_Evangelidis(): yco = %s" % \ str(yco)); # Note: even for HD frames, xco.shape = (11, 11) (yco the same) #arg = -(xco.*xco + yco.*yco) / (2*sd*sd) #arg = -(xco * xco + yco * yco) / (2.0 * sd * sd); arg = -(xco ** 2 + yco ** 2) / (2.0 * sd * sd); if common.MY_DEBUG_STDOUT: common.DebugPrint("multi_scale_harris_Evangelidis(): arg = %s" % \ str(arg)); #%2d gaussian kernel """ From http://www.mathworks.com/help/matlab/ref/exp.html: "exp(X) returns the exponential for each element of array X." """ #g=exp(arg)/(2*pi*sd^2); #2d gaussian kernel g = np.exp(arg) / (2.0 * math.pi * pow(sd, 2)); if common.MY_DEBUG_STDOUT: common.DebugPrint("multi_scale_harris_Evangelidis(): g = %s" % \ str(g)); # normalize to suppress any gain #if sum(g(:))~=0: g_sum = g.sum(); #if abs(g.sum()) > 1.0e-6: if abs(g_sum) > 1.0e-6: #g = g / sum(g(:)); #g = g / float(g.sum()); g /= float(g_sum); if common.MY_DEBUG_STDOUT: common.DebugPrint("multi_scale_harris_Evangelidis(): sd = %s" % str(sd)); common.DebugPrint("multi_scale_harris_Evangelidis(): w = %s" % str(w)); """ #%Instead of computing derivatives in the filtered image, we filter the image with the derivatives of the kernel. """ #% kernels for gaussian derivatives #gx=-xco.*g/(sd*sd); gx = -xco * g / float(sd * sd); #gy=-yco.*g/(sd*sd); gy = -yco * g / float(sd * sd); if common.MY_DEBUG_STDOUT: common.DebugPrint("multi_scale_harris_Evangelidis(): gx = %s" % \ str(gx)); common.DebugPrint("multi_scale_harris_Evangelidis(): gy = %s" % \ str(gy)); """ multi_scale_harris_Evangelidis(): arg.shape = (11, 11) multi_scale_harris_Evangelidis(): g.shape = (11, 11) multi_scale_harris_Evangelidis(): gx.shape = (11, 11) multi_scale_harris_Evangelidis(): gy.shape = (11, 11) multi_scale_harris_Evangelidis(): gi.shape = (15, 15) """ common.DebugPrint("multi_scale_harris_Evangelidis(): xco.shape = %s" % str(xco.shape)); common.DebugPrint("multi_scale_harris_Evangelidis(): yco.shape = %s" % str(yco.shape)); common.DebugPrint("multi_scale_harris_Evangelidis(): arg.shape = %s" % str(arg.shape)); common.DebugPrint("multi_scale_harris_Evangelidis(): g.shape = %s" % str(g.shape)); common.DebugPrint("multi_scale_harris_Evangelidis(): gx.shape = %s" % str(gx.shape)); common.DebugPrint("multi_scale_harris_Evangelidis(): gy.shape = %s" % str(gy.shape)); #%compute the derivatives #Ix = imfilter(im, gx, 'replicate'); Ix = Matlab.imfilter(im, gx, "replicate"); #Iy = imfilter(im, gy, 'replicate'); Iy = Matlab.imfilter(im, gy, "replicate"); #% Alex: Ix and Iy have the same size as im #if True: if common.MY_DEBUG_STDOUT: common.DebugPrint("multi_scale_harris_Evangelidis(): Ix = %s" % \ str(Ix)); common.DebugPrint("multi_scale_harris_Evangelidis(): Iy = %s" % \ str(Iy)); #% gaussian kernel to compute 2nd moment matrix #if mod(floor(6*si),2) %size: six times the integration scale if int(math.floor(6 * si)) % 2 == 1: #size: six times the integration scale #gi = fspecial('ga',max(1,fix(6*si)), si) gi = Matlab.fspecial("ga", max(1, Matlab.fix(6 * si)), si); else: #gi = fspecial('ga',max(1,fix(6*si)+1), si) gi = Matlab.fspecial("ga", max(1, Matlab.fix(6 * si) + 1), si); #if True: if common.MY_DEBUG_STDOUT: common.DebugPrint("multi_scale_harris_Evangelidis(): gi = %s" % \ str(gi)); common.DebugPrint("multi_scale_harris_Evangelidis(): gi.shape = %s" % \ str(gi.shape)); #Ix2 = imfilter(Ix.^2, gi, 'replicate'); Ix2 = Matlab.imfilter(Ix ** 2, gi, "replicate"); #Iy2 = imfilter(Iy.^2, gi, 'replicate'); Iy2 = Matlab.imfilter(Iy ** 2, gi, "replicate"); #Ixy = imfilter(Ix.*Iy, gi, 'replicate'); Ixy = Matlab.imfilter(Ix * Iy, gi, "replicate"); #% Alex: Ix2, Iy2 and Ixy have the same size as im #if True: if common.MY_DEBUG_STDOUT: common.DebugPrint("multi_scale_harris_Evangelidis(): Ix2 = %s" % \ str(Ix2)); common.DebugPrint("multi_scale_harris_Evangelidis(): Iy2 = %s" % \ str(Iy2)); common.DebugPrint("multi_scale_harris_Evangelidis(): Ixy = %s" % \ str(Ixy)); common.DebugPrint("multi_scale_harris_Evangelidis(): Ix2.dtype = %s" % \ str(Ix2.dtype)); common.DebugPrint("multi_scale_harris_Evangelidis(): Iy2.dtype = %s" % \ str(Iy2.dtype)); common.DebugPrint("multi_scale_harris_Evangelidis(): Ixy.dtype = %s" % \ str(Ixy.dtype)); #%% Cornerness measure #% Noble measure. #% #% M = (Ix2.*Iy2 - Ixy.^2)./(Ix2 + Iy2 + eps); #% Harris measure. #M = (Ix2.*Iy2 - Ixy.^2) - .06*(Ix2 + Iy2).^2; M = (Ix2 * Iy2 - Ixy ** 2) - 0.06 * (Ix2 + Iy2) ** 2; if common.MY_DEBUG_STDOUT: common.DebugPrint("multi_scale_harris_Evangelidis(): M.dtype = %s" % \ str(M.dtype)); common.DebugPrint("multi_scale_harris_Evangelidis(): M = %s" % \ str(M)); #% Alex: scn is a vector - see definition above #% Alex: M has the same size as im #M = scn(i)*M; M = scn[i - 1] * M; #thresh = 0.001*max(M(:)); thresh = 0.001 * M.max(); #% imagesc(M==abs(M));axis on; #% colorbar #% pause """ %keep points that are the maximum in a neighborhood of radius=round(3*si/2) and are above thresh %non-maximum supression and subpixel refinement """ #[r,c, rsubp, csubp] = my_nms(M, round(3*si/2), thresh); r, c, rsubp, csubp = my_nms(M, round(3 * si / 2.0), thresh); if common.MY_DEBUG_STDOUT: common.DebugPrint("multi_scale_harris_Evangelidis(): r.shape = %s" % str(r.shape)); common.DebugPrint("multi_scale_harris_Evangelidis(): r = %s" % str(r)); common.DebugPrint("multi_scale_harris_Evangelidis(): c.shape = %s" % str(c.shape)); common.DebugPrint("multi_scale_harris_Evangelidis(): c = %s" % str(c)); common.DebugPrint("multi_scale_harris_Evangelidis(): rsubp.shape = %s" % str(rsubp.shape)); common.DebugPrint("multi_scale_harris_Evangelidis(): rsubp = %s" % str(rsubp)); common.DebugPrint("multi_scale_harris_Evangelidis(): csubp.shape = %s" % str(csubp.shape)); common.DebugPrint("multi_scale_harris_Evangelidis(): csubp = %s" % str(csubp)); #% Alex: r,c, rsubp, csubp seem to always be the same size - and the #% size I've seen is 56 * 1???? #pp=[rsubp, csubp, i*ones(size(r,1),1)]; pp = np.c_[rsubp, csubp, i * np.ones((r.shape[0], 1))]; #% Alex: here we add more rows (pp) to points below the existing rows of #% points #points=[points; pp]; common.DebugPrint("multi_scale_harris_Evangelidis(): points.shape = %s" % str(points.shape)); common.DebugPrint("multi_scale_harris_Evangelidis(): pp.shape = %s" % str(pp.shape)); if points.size == 0: # Avoiding exception: "ValueError: arrays must have same number of dimensions" points = pp; else: points = np.r_[points, pp]; #toc if disp: assert False; # not implemented the display of Harris features """ figure; imshow(im,[]) #hold on title('corners detected') for i = range(1, size(points,1) + 1): rectangle('Position',[points(i,2)-3*points(i,3),points(i,1)-3*points(i,3),... 2*3*points(i,3),2*3*points(i,3)],'Curvature',[1,1],'EdgeColor','w','LineWidth',2) plot(points(i,2),points(i,1),'r+') """ if common.MY_DEBUG_STDOUT: common.DebugPrint("multi_scale_harris_Evangelidis(): points = %s" % str(points)); common.DebugPrint("multi_scale_harris_Evangelidis(): points.shape = %s" % str(points.shape)); tMSH2 = float(cv2.getTickCount()); myTime = (tMSH2 - tMSH1) / cv2.getTickFrequency(); print("multi_scale_harris_Evangelidis(): multi_scale_harris_Evangelidis() took %.6f [sec]" % myTime); return points;
def multiscale_quad_retrieval(r_quadsTree, r_harlocs, q_harlocs, md_threshold, st_threshold, \ all_ori, all_id, all_max, all_cen, nos, scale_index, cropflag, \ sequence): common.DebugPrint("Entered multiscale_quad_retrieval(): " \ "md_threshold = %s, st_threshold = %s." % \ (str(md_threshold), \ str(st_threshold))) assert len(r_harlocs) != 0 assert len(q_harlocs) != 0 try: Votes_space = np.load("Votes_space%d.npz" % scale_index)['arr_0'] HH = np.load("HH%d.npz" % scale_index)['arr_0'] return Votes_space, HH except: common.DebugPrintErrorTrace() if common.MY_DEBUG_STDOUT and DBGPRINT: common.DebugPrint("multiscale_quad_retrieval(): r_quadsTree = %s" % \ str(r_quadsTree)) common.DebugPrint("multiscale_quad_retrieval(): len(r_harlocs) = %d" % len(r_harlocs)) common.DebugPrint("multiscale_quad_retrieval(): r_harlocs = %s" % str(r_harlocs)) common.DebugPrint("multiscale_quad_retrieval(): q_harlocs = %s" % str(q_harlocs)) common.DebugPrint("multiscale_quad_retrieval(): md_threshold = %s" % str(md_threshold)) print("multiscale_quad_retrieval(): st_threshold = %s" % str(st_threshold)) #common.DebugPrint("multiscale_quad_retrieval(): all_ori, all_id, all_max, all_cen, nos, scale_index, cropflag = %s" % str(all_ori, all_id, all_max, all_cen, nos, scale_index, cropflag)); common.DebugPrint("multiscale_quad_retrieval(): all_id = %s" % str(all_id)) common.DebugPrint("multiscale_quad_retrieval(): all_id.shape = %s" % (str(all_id.shape))) #common.DebugPrint("multiscale_quad_retrieval(): all_max, all_cen, nos, scale_index, cropflag = %s" % str(all_max, all_cen, nos, scale_index, cropflag)); #common.DebugPrint("multiscale_quad_retrieval(): all_max = %s" % str(all_max)); #common.DebugPrint("multiscale_quad_retrieval(): all_cen, nos, scale_index, cropflag = %s" % str(all_cen, nos, scale_index, cropflag)); common.DebugPrint("multiscale_quad_retrieval(): sequence = %s" % str(sequence)) print("multiscale_quad_retrieval(): cropflag = %s" % str(cropflag)) t1 = float(cv2.getTickCount()) if scale_index > nos: assert scale_index <= nos #error('Wrong scale index or number-of-scales'); #QD = dir([q_path "multiharlocs*.mat"]) #QD = [q_path + "multiharlocs*.mat"] #QD = q_harlocs; #RD = dir([r_path "multiharlocs*.mat"]) #RD = [r_path + "multiharlocs*.mat"] #RD = r_harlocs; #TODO: take out RD_start #RD_start = str2num(RD(1).name(end - 9 : end - 4)) #RD_start = int(RD[0][-9 : -4]) RD_start = 0 #RD_end = str2num(RD(end).name(end - 9 : end - 4)) #RD_end = int(RD[-1][-9 : -4]) #RD_end = len(RD) - 1; RD_end = len(r_harlocs) - 1 if False: # n_d not used anywhere #n_d = hist(all_id, RD_start : RD_end) #n_d = hist[all_id, RD_start : RD_end] n_d = Matlab.hist(x=all_id, \ binCenters=np.array(range(RD_start, RD_end + 1)) ) #cross_indices = np.zeros( (len(QD), 2) ); cross_indices = np.zeros((len(q_harlocs), 2)) j = 1 #tic #ORI = np.array([]); # ORI NOT used anywhere """ Inspired from https://stackoverflow.com/questions/17559140/matlab-twice-as-fast-as-numpy BUT doesn't help in this case: Votes_space = np.asfortranarray(np.zeros( (len(RD), len(QD)) )); """ #Votes_space = np.zeros( (len(RD), len(QD)) ); Votes_space = np.zeros((len(r_harlocs), len(q_harlocs))) # Make a distinct copy of HH from Votes_space... #HH = Votes_space.copy().astype(np.int16); #Votes_space + 0; #HH = np.zeros((len(RD), len(QD)), dtype=np.int8); HH = np.zeros((len(r_harlocs), len(q_harlocs)), dtype=np.int8) #!!!!TODO use MAYBE even np.bool - OR take it out #common.DebugPrint("multiscale_quad_retrieval(): Votes_space = %s,\n HH = %s" % (str(Votes_space), str(HH))) tolers = 0.1 - float(scale_index) / 100.0 # it helps to make more strict the threshold as the scale goes up # tolers = 0.15 - float(scale_index) / 100.0; MAXDIS = 3 + scale_index MAXORI = 0.25 """ !!!!TODO TODO: I am using multiprocessing.Poll and return votes; the dispatcher assembles the results, but the results are NOT the same with the serial case - although they look pretty decent, but they seem to be suboptimal - dp_Alex returns suboptimal cost path for USE_MULTITHREADING == True instead of False. (Note: running under the same preconditions multiscale_quad_retrieval I got the same results in dp_Alex(). """ if False: #config.USE_MULTITHREADING == True: global g g.r_quadsTree = r_quadsTree g.r_harlocs = r_harlocs g.q_harlocs = q_harlocs g.md_threshold = md_threshold g.st_threshold = st_threshold g.all_ori = all_ori g.all_id = all_id g.all_max = all_max g.all_cen = all_cen g.nos = nos g.scale_index = scale_index g.cropflag = cropflag g.sequence = sequence g.RD_start = RD_start g.RD_end = RD_end g.MAXDIS = MAXDIS g.MAXORI = MAXORI g.tolers = tolers """ Start worker processes to use on multi-core processor (able to run in parallel - no GIL issue if each core has it's own VM) """ pool = multiprocessing.Pool(processes=config.numProcesses) print("multiscale_quad_retrieval(): Spawned a pool of %d workers" % \ config.numProcesses) listParams = range(0, len(q_harlocs)) #!!!!TODO: use counterStep, config.initFrame[indexVideo] #res = pool.map(IterationStandaloneMQR, listParams); # See https://docs.python.org/2/library/multiprocessing.html#module-multiprocessing.pool res = pool.map(func=IterationStandaloneMQR, iterable=listParams, \ chunksize=1) print("Pool.map returns %s" % str(res)) #x0.size + 1 """ From https://medium.com/building-things-on-the-internet/40e9b2b36148 close the pool and wait for the work to finish """ pool.close() pool.join() # Doing the "reduce" phase after the workers have finished :) assert len(res) == len(q_harlocs) for queryFrame, resE in enumerate(res): resEIndex = resE[0] resE = resE[1] assert resEIndex == queryFrame # Gives: "ValueError: output operand requires a reduction, but reduction is not enabled" #Votes_space[:, queryFrame - 1] = votes; Votes_space[:, queryFrame] = resE for queryFrame in range(len(q_harlocs)): if cropflag == 0: HH[:, queryFrame] = 1 else: """ HH[:, queryFrame] = spatial_consistency.spatial_consistency(space_xy, \ qcen, len(RD), st_threshold, cropflag); """ HH[:, queryFrame] = spatial_consistency.spatial_consistency(space_xy, \ qcen, len(r_harlocs), st_threshold, cropflag) try: np.savez_compressed("Votes_space%d" % scale_index, Votes_space) np.savez_compressed("HH%d" % scale_index, HH) except: common.DebugPrintErrorTrace() return Votes_space, HH """ We substitute q - 1 with q, since we want to number arrays from 0 (not from 1 like in Matlab). """ #for q=1:length(QD) #for q in range(1, len(QD) + 1): #for queryFrame in range(len(QD)): for queryFrame in range(len(q_harlocs)): common.DebugPrint( "multiscale_quad_retrieval(): Starting iteration queryFrame = %d" % queryFrame) # tic """ str1=['load ' q_path QD(q).name] eval(str1) """ """ We make pp reference the desired multiharloc list for the query video frame queryFrame """ pp = q_harlocs[queryFrame] #pp = np.array(pp); #common.DebugPrint("multiscale_quad_retrieval(): pp = %s" % str(pp)); #[qout,qcen,qmaxdis,qori]=findquads(pp(pp(:,3)==scale_index,1:2),md_threshold,0); points = pp[pp[:, 2] == scale_index, 0:2] qout, qcen, qmaxdis, qori = findquads.findquads( points, md_threshold, 0) if common.MY_DEBUG_STDOUT and DBGPRINT: print("multiscale_quad_retrieval(): queryFrame = %d, " \ "qout.shape (number of quads for query frame queryFrame) = %s" % \ (queryFrame, str(qout.shape))) # disp([num2str(q) ' of ' num2str(length(QD)) ' -> ' num2str(size(qout,1)) ' quads']) #space_xy=zeros(size(qcen,1),2*length(RD))+nan; #space_xy = np.zeros( (qcen.shape[0], 2 * len(RD)) ) + np.nan; space_xy = np.zeros((qcen.shape[0], 2 * len(r_harlocs))) + np.nan # votes=zeros(length(RD),1) #votes=zeros(length(RD),length(tolers)); #votes = np.zeros( (len(RD), 1) ); votes = np.zeros((len(r_harlocs), 1)) #nep = np.array([]); #m_points = np.array([]); assert isinstance(tolers, float) if common.MY_DEBUG_STDOUT: common.DebugPrint( "multiscale_quad_retrieval(): quads of query frame %d are: " % queryFrame) common.DebugPrint(" qout = %s" % str(qout)) """ Alex: for each quad (4 floats) of the query frame from Harris feature of scale scale_index Note: all_id stores the reference frame id for each quad descriptor. """ """ We substitute queryFrameQuad - 1 with queryFrameQuad, since we want to number arrays from 0 (not from 1 like in Matlab). """ #for queryFrameQuad in range(1, qout.shape[0] + 1): for queryFrameQuad in range(qout.shape[0]): common.DebugPrint( "multiscale_quad_retrieval(): Starting iteration queryFrameQuad = %d" % queryFrameQuad) """ Matlab's polymorphism is really bugging here: although it's normally a float, tolers is considered to be a size 1 vector... so len(tolers) == 1 """ #for tol_i in range(1, len(tolers) + 1): # tol = tolers[tol_i - 1] """ We substitute tol_i - 1 with tol, since we want to number arrays from 0 (not from 1 like in Matlab). """ #for tol_i in range(1, 1 + 1): for tol_i in range(1): tol = tolers """ # TODO: done below - take out this dbg print if DBGPRINT: common.DebugPrint("multiscale_quad_retrieval(): " \ "qout[queryFrameQuad, :] = %s" % \ str(qout[queryFrameQuad, :])) """ #% default for first PAMI with tol= 0.1 approximately # NOTE: SciPy's KDTree finds a few more results, in some cases, # than the Matlab code from Evangelidis. #idx, di = kdtree_ball_query(tree, qout(i, :), tol) #idx, distKD = kdtree_ball_query(tree, qout[i - 1, :], tol) #idx, di = tree.query(x=xQuery, k=4) #resPoints = [data[i] for i in resBallIndices] # tol is a scalar representing the radius of the ball if config.KDTREE_IMPLEMENTATION == 0: idx = r_quadsTree.query_ball_point(qout[queryFrameQuad, :], tol) elif config.KDTREE_IMPLEMENTATION == 1: #pt = qout[queryFrameQuad - 1, :].astype(np.float32); pt = qout[queryFrameQuad, :] pt = np.array([[pt[0], pt[1], pt[2], pt[3]]], dtype=np.float32) retval, idx, dists = r_quadsTree.radiusSearch( \ query=pt, \ radius=(tol**2), \ maxResults=NUM_MAX_ELEMS, \ params=search_params) if common.MY_DEBUG_STDOUT and DBGPRINT: """ common.DebugPrint("multiscale_quad_retrieval(): " \ "retval (number NNs) = %s" % str(retval)); """ common.DebugPrint( \ "multiscale_quad_retrieval(): radiusSearch's retval " \ "(at queryFrame=%d, queryFrameQuad=%d) is %d" % (queryFrame, queryFrameQuad, retval)) idx = idx[0] dists = dists[0] """ Note: retval is the number of neighbors returned from the radiusSearch(). But the idx and the dists can have more elements than the returned retval. """ idx = idx[:retval] dists = dists[:retval] if common.MY_DEBUG_STDOUT and DBGPRINT: print("multiscale_quad_retrieval(): " \ "qout[queryFrameQuad, :] = %s" % str(qout[queryFrameQuad, :])) print("multiscale_quad_retrieval(): " \ "idx = %s" % str(idx)) print("multiscale_quad_retrieval(): " \ "dists = %s" % str(dists)) print("multiscale_quad_retrieval(): " \ "tol = %s" % str(tol)) if config.KDTREE_IMPLEMENTATION == 0: print("multiscale_quad_retrieval(): " \ "r_quadsTree.data[idx] = %s" % \ str(r_quadsTree.data[idx])) # We print the distances to the points returned in idx if common.MY_DEBUG_STDOUT and DBGPRINT: # This is just for debugging purposes a = qout[queryFrameQuad, :] if config.KDTREE_IMPLEMENTATION == 0: for myI, index in enumerate(idx): b = r_quadsTree.data[index] """ if False: common.DebugPrint("multiscale_quad_retrieval(): distance to " \ "%d point (%s) inside ball = %.4f" % \ (myI, str(b), npla.norm(a - b))); """ else: pass idx = np.array(idx) #if False: if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " \ "all_max.shape = %s" % str(all_max.shape)) common.DebugPrint("multiscale_quad_retrieval(): " \ "qmaxdis.shape = %s" % str(qmaxdis.shape)) common.DebugPrint("multiscale_quad_retrieval(): " \ "qmaxdis = %s" % str(qmaxdis)) common.DebugPrint("multiscale_quad_retrieval(): " \ "qori.shape = %s" % str(qori.shape)) common.DebugPrint("multiscale_quad_retrieval(): " \ "qori = %s" % str(qori)) #dis_idx=abs(qmaxdis(i)-all_max(idx))<MAXDIS; if len(idx) == 0: # NOT A GOOD IDEA: continue; #idx = np.array([]); dis_idx = np.array([]) ori_idx = np.array([]) else: if common.MY_DEBUG_STDOUT and DBGPRINT: print("multiscale_quad_retrieval(): " \ "queryFrameQuad = %s" % str(queryFrameQuad)) print("multiscale_quad_retrieval(): " \ "all_max[idx] = %s" % str(all_max[idx])) print("multiscale_quad_retrieval(): " \ "qmaxdis[queryFrameQuad] = %s" % str(qmaxdis[queryFrameQuad])) if USE_GPS_COORDINATES: # We look only at a part of the reference video """ Since in some cases the video temporal alignment is difficult to do due to similar portions in the trajectory (see the drone videos, clip 3_some_lake) we "guide" the temporal alignment by restricting the reference frame search space - this is useful when we have the geolocation (GPS) coordinate for each frame. """ if common.MY_DEBUG_STDOUT and DBGPRINT: print("multiscale_quad_retrieval(): " \ "all_id = %s" % str(all_id)) if True: #assert (all_id.ndim == 2) and (all_id.shape[1] == 1); if all_id.ndim == 2: #!!!!TODO TODO: put this at the beginning of the function assert all_id.shape[1] == 1 """ We flatten the array all_id Note: We don't use order="F" since it's basically 1-D array """ all_id = np.ravel(all_id) #!!!!TODO: put start and end frame in config - or compute it from geolocation sub_idx = np.logical_and( (all_id[idx] >= 2030 - 928), \ (all_id[idx] <= 2400 - 928) ) idx = idx[sub_idx] if common.MY_DEBUG_STDOUT and DBGPRINT: print("multiscale_quad_retrieval(): " \ "all_id = %s" % str(all_id)) print("multiscale_quad_retrieval(): " \ "sub_idx = %s" % str(sub_idx)) print("multiscale_quad_retrieval(): " \ "idx = %s" % str(idx)) if FILTER: dis_idx = np.abs(qmaxdis[queryFrameQuad] - all_max[idx]) < MAXDIS #if False: if common.MY_DEBUG_STDOUT: """ common.DebugPrint("multiscale_quad_retrieval(): " \ "idx = %s" % str(idx)); """ common.DebugPrint("multiscale_quad_retrieval(): " \ "dis_idx = %s" % str(dis_idx)) #idx=idx(dis_idx) idx = idx[dis_idx] #if False: if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " \ "idx (after idx = idx[dis_idx]) = %s" % str(idx)) if FILTER: #ori_idx=abs(qori(i)-all_ori(idx))<MAXORI; ori_idx = np.abs(qori[queryFrameQuad] - all_ori[idx]) < MAXORI #if False: if common.MY_DEBUG_STDOUT: """ common.DebugPrint("multiscale_quad_retrieval(): " \ "all_ori = %s" % str(all_ori)); common.DebugPrint("multiscale_quad_retrieval(): " \ "qori[queryFrameQuad] = %s" % str(qori[queryFrameQuad])); """ common.DebugPrint("multiscale_quad_retrieval(): " \ "ori_idx = %s" % str(ori_idx)) #idx=idx(ori_idx); idx = idx[ori_idx] # IMPORTANT ################################################### # IMPORTANT ################################################### # IMPORTANT ################################################### #% spatio-temporal consistency # IMPORTANT ################################################### # IMPORTANT ################################################### # IMPORTANT ################################################### #if numel(idx) > 0: if idx.size > 0: if cropflag == 0: if FILTER: """ Alex: this is a simple procedure of eliminating False Positive (FP) matches, as presented in Section 4.2 of TPAMI 2013 paper. Basically it filters out quad matches that have centroids st_threshold away from the query quad. Note: all_cen are the controids of all reference quads. """ dy = qcen[queryFrameQuad, 0] - all_cen[idx, 0] dx = qcen[queryFrameQuad, 1] - all_cen[idx, 1] #D=dy.^2+dx.^2; D = dy**2 + dx**2 co_idx = D < pow(st_threshold, 2) idx = idx[co_idx] else: """ We substitute iii - 1 with iii, since we want to number arrays from 0 (not from 1 like in Matlab). """ #for iii in range(1, len(idx) + 1): for iii in range(len(idx)): #space_xy(i,(all_id(idx(iii))-RD_start)*2+1:(all_id(idx(iii))-RD_start)*2+2) = all_cen(idx(iii),:) space_xy[queryFrameQuad, \ (all_id[idx[iii]] - RD_start) * 2: (all_id[idx[iii] - 1] - RD_start) * 2 + 1] = \ all_cen[idx[iii], :] #hh=hist(all_id(idx),RD_start:RD_end); # It has to be an np.array because we multiply it with a scalar histoRange = np.array(range(RD_start, RD_end + 1)) hh = Matlab.hist(x=all_id[idx], binCenters=histoRange) #if False: #if True: if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " \ "hh = %s" % (str(hh))) common.DebugPrint("multiscale_quad_retrieval(): " \ "hh.shape = %s" % (str(hh.shape))) """ common.DebugPrint("multiscale_quad_retrieval(): " \ "all_id = %s" % (str(all_id))); """ common.DebugPrint("multiscale_quad_retrieval(): " \ "all_id.shape = %s" % (str(all_id.shape))) common.DebugPrint("multiscale_quad_retrieval(): " \ "idx = %s" % (str(idx))) common.DebugPrint("multiscale_quad_retrieval(): " \ "idx.shape = %s" % (str(idx.shape))) # % nz can be computed more optimally #nz=find(hh~=0); # nz can be computed more optimally # np.nonzero() always returns a tuple, even if it contains 1 element since hh has only 1 dimension nz = np.nonzero(hh != 0)[0] #if False: if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " \ "nz = %s" % (str(nz))) common.DebugPrint("multiscale_quad_retrieval(): " \ "nz.shape = %s" % (str(nz.shape))) #if numel(nz) > 0 if nz.size > 0: #%%----text-retrieval-like #votes(nz, tol_i) = votes(nz, tol_i) + log10(length(RD) / (length(nz)))^2 #PREVIOUSLY #myVal = pow(math.log10(float(len(RD)) / len(nz)), 2); myVal = pow( math.log10(float(len(r_harlocs)) / len(nz)), 2) """ try: myVal = pow(math.log10(float(len(r_harlocs)) / len(nz)), 2); except: print("Error: len=%d len(nz)=%d nz.size=%d" % \ (len(r_harlocs), len(nz), nz.size)); common.DebugPrintErrorTrace(); """ #if False: if common.MY_DEBUG_STDOUT: """ common.DebugPrint("multiscale_quad_retrieval(): " \ "len(RD) = %d" % len(RD)); """ common.DebugPrint("multiscale_quad_retrieval(): " \ "len(r_harlocs) = %d" % len(r_harlocs)) common.DebugPrint("multiscale_quad_retrieval(): " \ "len(nz) = %d" % len(nz)) common.DebugPrint("multiscale_quad_retrieval(): " \ "myVal = %.5f" % myVal) # PREVIOUSLY votes[nz, tol_i] = votes[nz, tol_i] + myVal # votes(nz)=votes(nz)+log10(length(RD)/(length(nz))); # votes(nz)=votes(nz)+1; if common.MY_DEBUG_STDOUT and DBGPRINT: """ common.DebugPrint("multiscale_quad_retrieval(): " \ "Votes_space.shape = %s" % (str(Votes_space.shape))); common.DebugPrint("multiscale_quad_retrieval(): " \ "votes.shape = %s" % (str(votes.shape))); """ print("multiscale_quad_retrieval(): " \ "votes.shape = %s" % (str(votes.shape))) if (np.abs(votes) < 1.0e-10).all(): print( \ "multiscale_quad_retrieval(): votes = 0 (all zeros)") else: print("multiscale_quad_retrieval(): " \ "votes = %s" % (str(votes))) #Votes_space(:,q)=votes; # Gives: "ValueError: output operand requires a reduction, but reduction is not enabled" #Votes_space[:, queryFrame - 1] = votes; # Note: since votes is basically a 1-D vector, we don't use the Fortran order Votes_space[:, queryFrame] = np.ravel(votes) # order="F"); if cropflag == 0: HH[:, queryFrame] = 1 else: """ HH[:, queryFrame] = spatial_consistency.spatial_consistency(space_xy, \ qcen, len(RD), st_threshold, cropflag); """ HH[:, queryFrame] = spatial_consistency.spatial_consistency(space_xy, \ qcen, len(r_harlocs), st_threshold, cropflag) if common.MY_DEBUG_STDOUT and DBGPRINT: print("multiscale_quad_retrieval(scale_index=%d): " \ "Votes_space =\n%s" % (scale_index, str(Votes_space))) try: np.savez_compressed("Votes_space%d" % scale_index, Votes_space) np.savez_compressed("HH%d" % scale_index, HH) except: common.DebugPrintErrorTrace() t2 = float(cv2.getTickCount()) myTime = (t2 - t1) / cv2.getTickFrequency() print("multiscale_quad_retrieval() took %.6f [sec]" % myTime) """ common.DebugPrint("multiscale_quad_retrieval(): " \ "%d corresponding frames retrieved in %.6f secs" % \ (len(q_harlocs), myTime)); """ return Votes_space, HH
def dp3(Vspace, numFramesR, numFramesQ, BOV_flag): #% Dynamic programming for a maximum-vote path in vote-space #% 2010, Georgios Evangelidis <*****@*****.**> print ("Entered dp3(): Running dynamic programming...") if common.MY_DEBUG_STDOUT: common.DebugPrint("dp3(): Vspace = %s" % str(Vspace)) # tic # QD=dir([q_path 'multiharlocs*.mat']); # RD=dir([r_path 'multiharlocs*.mat']); # cross=zeros(length(QD),2); # crossref = np.zeros( (len(q_path), 2) ); crossref = np.zeros((numFramesQ, 2)) # sigma_0=1.2; sigma_0 = 1.2 # [r,c,d] = size(Vspace); r, c, d = Vspace.shape # n=[1:d]; %scale levels n = np.array(range(1, d + 1)) #%scale levels # sigma_I=sqrt(1.8).^n*sigma_0; sigma_I = math.sqrt(1.8) ** n * sigma_0 w = sigma_I #%w=w*0+1; # w=w/sum(w); w = w / float(w.sum()) if BOV_flag == 1: # w=fliplr(w); w = w[:, ::-1] #% Initialization # D = zeros(r+1, c+1); D = np.zeros((r + 1, c + 1)) # D(1,:) = NaN; D[0, :] = np.nan # D(:,1) = NaN; D[:, 0] = np.nan # D(1,1) = 0; D[0, 0] = 0 # for i=1:d """ We substitute i - 1 with i since arrays start with 0 in Python, not with 1 like in Matlab. """ # for i in range(1, d + 1): for i in range(d): # V_temp=Vspace(:,:,i); V_temp = Vspace[:, :, i] # V_temp(isnan(V_temp))=0; V_temp[np.isnan(V_temp)] = 0 # !!!!TODO: check OK # Vspace(:,:,i)=V_temp; Vspace[:, :, i] = V_temp # VV=zeros(r,c); VV = np.zeros((r, c)) # for j=1:d """ We substitute j - 1 with j since arrays start with 0 in Python, not with 1 like in Matlab. """ # for j in range(1, d + 1): for j in range(d): # VV=VV+Vspace(:,:,j); VV = VV + Vspace[:, :, j] # D(2:end,2:end)=VV; D[1:, 1:] = VV # NEW_DP3_ALEX = True; NEW_DP3_ALEX = False if NEW_DP3_ALEX: # Alex: added cost cost = np.zeros((r + 1, c + 1)) #% for traceback # Tback = zeros(r+1,c+1); Tback = np.zeros((r + 1, c + 1)) # if True: if common.MY_DEBUG_STDOUT: common.DebugPrint("dp3(): printing locally optimum solutions:") # Alex: trying out to find a better solution than dp3() !!!!TODO : more # This solution is basically the one returned by causal() IF we do NOT apply Matlab.filter2() on Vspace for j in range(1, c + 1): maxCol = 0.0 maxPos = -1 for i in range(1, r + 1): assert D[i, j] >= 0.0 if maxCol < D[i, j]: maxCol = D[i, j] maxPos = i # So for query frame j we have a candidate matching ref frame i common.DebugPrint("dp3(): for query frame %d - candidate frame %d" % (j - 1, maxPos)) common.DebugPrint("dp3(): for query frame %d we found matching ref frame %d" % (j - 1, maxPos)) # !!!!TODO: make i =0.., j=0.. and substitute i-1 with i, i-2 with i-1, etc # for i = 1:r; for i in range(1, r + 1): # for j = 1:c for j in range(1, c + 1): # if (i>1) && (j>1) if (i > 1) and (j > 1): # dd1 = w(1)*Vspace(i-1,max(1,j-2),1); dd1 = w[0] * Vspace[i - 2, max(0, j - 3), 0] # dd2 = w(1)*Vspace(max(1,i-2),j-1,1); dd2 = w[0] * Vspace[max(0, i - 3), j - 2, 0] # dd3 = w(1)*Vspace(i-1,j-1,1); dd3 = w[0] * Vspace[i - 2, j - 2, 0] # dd4 = w(1)*Vspace(i-1,j,1); dd4 = w[0] * Vspace[i - 2, j - 1, 0] # dd5 = w(1)*Vspace(i,j-1,1); dd5 = w[0] * Vspace[i - 1, j - 2, 0] if d > 1: # for sc = 2:d for sc in range(2, d + 1): # dd1 = max(dd1, w(sc)*Vspace(i-1,max(1,j-2),sc)); dd1 = max(dd1, w[sc - 1] * Vspace[i - 2, max(0, j - 3), sc - 1]) # dd2 = max(dd2, w(sc)*Vspace(max(1,i-2),j-1,sc)); dd2 = max(dd2, w[sc - 1] * Vspace[max(0, i - 3), j - 2, sc - 1]) # dd3 = max(dd3, w(sc)*Vspace(i-1,j-1,sc)); dd3 = max(dd3, w[sc - 1] * Vspace[i - 2, j - 2, sc - 1]) # dd4 = max(dd4, w(sc)*Vspace(i-1,j,sc)); dd4 = max(dd4, w[sc - 1] * Vspace[i - 2, j - 1, sc - 1]) # dd5 = max(dd5, w(sc)*Vspace(i,j-1,sc)); dd5 = max(dd5, w[sc - 1] * Vspace[i - 1, j - 2, sc - 1]) # D(i,j-1)=D(i,j-1)+dd1; D[i - 1, j - 2] += dd1 # D(i-1,j)=D(i-1,j)+dd2; D[i - 2, j - 1] += dd2 # D(i,j)=D(i,j)+ dd3; D[i - 1, j - 1] += dd3 # D(i,j+1)=D(i,j+1)+ dd4; D[i - 1, j] += dd4 # D(i+1,j)=D(i+1,j)+ dd5; D[i, j - 1] += dd5 #% % instead of above loop, use the following five lines when scales=6 #% D(i,j-1)=D(i,j-1)+max([w(1)*Vspace(i-1,max(1,j-2),1),w(2)*Vspace(i-1,max(1,j-2),2),w(3)*Vspace(i-1,max(1,j-2),3),w(4)*Vspace(i-1,max(1,j-2),4),w(5)*Vspace(i-1,max(1,j-2),5),w(6)*Vspace(i-1,max(1,j-2),6)]); #% D(i-1,j)=D(i-1,j)+max([w(1)*Vspace(max(1,i-2),j-1,1),w(2)*Vspace(max(1,i-2),j-1,2),w(3)*Vspace(max(1,i-2),j-1,3),w(4)*Vspace(max(1,i-2),j-1,4),w(5)*Vspace(max(1,i-2),j-1,5),w(6)*Vspace(max(1,i-2),j-1,6)]); #% D(i,j)=D(i,j)+max([w(1)*Vspace(i-1,j-1,1),w(2)*Vspace(i-1,j-1,2),w(3)*Vspace(i-1,j-1,3),w(4)*Vspace(i-1,j-1,4),w(5)*Vspace(i-1,j-1,5),w(6)*Vspace(i-1,j-1,6)]); #% D(i,j+1)=D(i,j+1)+max([w(1)*Vspace(i-1,j,1),w(2)*Vspace(i-1,j,2),w(3)*Vspace(i-1,j,3),w(4)*Vspace(i-1,j,4),w(5)*Vspace(i-1,j,5),w(6)*Vspace(i-1,j,6)]); #% D(i+1,j)=D(i+1,j)+max([w(1)*Vspace(i,j-1,1),w(2)*Vspace(i,j-1,2),w(3)*Vspace(i,j-1,3),w(4)*Vspace(i,j-1,4),w(5)*Vspace(i,j-1,5),w(6)*Vspace(i,j-1,6)]); #% [dmax, tb] = max([D(i, j), D(i-1, j), D(i, j-1), D(i,j+1),D(i+1,j)]); # [dmax, tb] = max([D(i, j)+1/sqrt(2), D(i-1, j)+1/sqrt(5), D(i, j-1)+1/sqrt(5), D(i,j+1)+1, D(i+1,j)+1]); dmax, tb = Matlab.max( np.array( [ D[i - 1, j - 1] + 1.0 / math.sqrt(2.0), D[i - 2, j - 1] + 1.0 / math.sqrt(5.0), D[i - 1, j - 2] + 1.0 / math.sqrt(5.0), D[i - 1, j] + 1, D[i, j - 1] + 1, ] ) ) else: # [dmax, tb] = max([D(i,j), D(i,j+1), D(i+1,j)]); dmax, tb = Matlab.max(np.array([D[i - 1, j - 1], D[i - 1, j], D[i, j - 1]])) if common.MY_DEBUG_STDOUT: common.DebugPrint("dp3(): dmax = %s" % str(dmax)) common.DebugPrint("dp3(): tb = %s" % str(tb)) # D(i+1,j+1) = D(i+1,j+1)+dmax; if NEW_DP3_ALEX: cost[i, j] = 0 #!!!!TODO: think more else: D[i, j] += dmax #!!!!TODO: for me it's weird he adds dmax here... # Tback(i+1,j+1) = tb; Tback[i, j] = tb if common.MY_DEBUG_STDOUT: common.DebugPrint("dp3(): D.shape = %s" % str(D.shape)) common.DebugPrint("dp3(): D = %s" % str(D)) common.DebugPrint("dp3(): Tback.shape = %s" % str(Tback.shape)) common.DebugPrint("dp3(): Tback = %s" % str(Tback)) #% Traceback i = r + 1 j = c + 1 y = i - 1 x = j - 1 # while i > 2 & j > 2 while (i > 2) and (j > 2): # tb = Tback(i,j); tb = Tback[i - 1, j - 1] + 1 # In Matlab, max returns indices from 1.. if tb == 1: i -= 1 j -= 1 elif tb == 2: i -= 2 j -= 1 elif tb == 3: i -= 1 j -= 2 elif tb == 4: i -= 1 j = j elif tb == 5: j -= 1 i = i else: # error; assert False # y = [i,y]; # NOT GOOD: y = np.c_[i, y]; y = np.hstack([i - 1, y]) # x = [j,x]; # NOT GOOD: x = np.c_[j, x]; x = np.hstack([j - 1, x]) if common.MY_DEBUG_STDOUT: common.DebugPrint("dp3(): before D.shape = %s" % str(D.shape)) common.DebugPrint("dp3(): before D = %s" % str(D)) #% Strip off the edges of the D matrix before returning # D = D(2:(r+1),2:(c+1)); D = D[1 : (r + 1), 1 : (c + 1)] if common.MY_DEBUG_STDOUT: common.DebugPrint("dp3(): D.shape = %s" % str(D.shape)) common.DebugPrint("dp3(): D = %s" % str(D)) # RD_start=str2num(RD(1).name(end-9:end-4)); RD_start = 1 if common.MY_DEBUG_STDOUT: common.DebugPrint("dp3(): Vspace.shape = %s" % str(Vspace.shape)) #!!!!TODO: understand well what is x,y and why computes p # for i=1:size(Vspace,2): for i in range(0, Vspace.shape[1]): # cross(i,1)=str2num(QD(i).name(end-9:end-4)); crossref[i, 0] = i # i; #!!!!TODO: think if OK # p=find(x==i); p = np.nonzero(x == i) p = p[0] if common.MY_DEBUG_STDOUT: common.DebugPrint("dp3(): x.shape = %s" % str(x.shape)) common.DebugPrint("dp3(): x = %s" % str(x)) common.DebugPrint("dp3(): y.shape = %s" % str(y.shape)) common.DebugPrint("dp3(): y = %s" % str(y)) common.DebugPrint("dp3(): i = %s" % str(i)) common.DebugPrint("dp3(): p = %s" % str(p)) # if isempty(p) if p.size == 0: #% Alex: Vali Codreanu said to change from temp=0; to temp=3; # temp=3; temp = 0 if common.MY_DEBUG_STDOUT: common.DebugPrint("dp3(): temp = %s" % str(temp)) crossref[i, 1] = 0 + RD_start - 1 else: # temp=y(p); temp = y[p] if common.MY_DEBUG_STDOUT: common.DebugPrint("dp3(): temp = %s" % str(temp)) # cross(i,2)=temp(end)+RD_start-1; if temp.size == 1: # If temp has only 1 element: crossref[i, 1] = temp + RD_start - 1 else: crossref[i, 1] = temp[-1] + RD_start - 1 # assert temp.size == 1; # common.DebugPrint("dp3(): temp = %s" % str(temp)); """ #cross(i,2)=temp(end)+RD_start-1; if True: crossref[i, 1] = temp[-1] + RD_start - 1; else: # BAD IDEA!!!!TODO crossref[i, 1] = temp + RD_start - 1; """ # printf("dp3(): Done in blabla secs\n"); common.DebugPrint("dp3(): crossref = %s" % str(crossref)) ComputeCost(crossref, VV, "crossref_dp3.txt") # return [y,x,D,Tback,cross] return y, x, D, Tback, crossref
def MyImageReadMSH(index): global g """ common.DebugPrint("MyImageReadMSH(): initFrame = %d)" % \ (config.initFrame[g.indexVideo])); index += config.initFrame[g.indexVideo]; """ """ common.DebugPrint("Entered MyImageReadMSH(capture=%s, index=%s)" % \ (str(capture), str(index))); """ common.DebugPrint("Entered MyImageReadMSH(index=%s)" % \ (str(index))) """ We must reopen the capture device in each different process, otherwise the program blocks at the first operation in the "global" capture device. """ if g.harlocsFolder.endswith(config.HARRIS_QUERY_FOLDER_NAME): if g.captureQ == None: capture = cv2.VideoCapture(sys.argv[1]) g.captureQ = capture common.DebugPrint("MyImageReadMSH(): new capture=%s" % \ (str(capture))) else: capture = g.captureQ elif g.harlocsFolder.endswith(config.HARRIS_REFERENCE_FOLDER_NAME): if g.captureR == None: capture = cv2.VideoCapture(sys.argv[2]) g.captureR = capture common.DebugPrint("MyImageReadMSH(): new capture=%s" % \ (str(capture))) else: capture = g.captureR else: assert False assert (g.indexVideo == 0) or (g.indexVideo == 1) if config.OCV_OLD_PY_BINDINGS: capture.set(cv2.cv.CV_CAP_PROP_POS_FRAMES, \ index) else: """ From http://docs.opencv.org/modules/highgui/doc/reading_and_writing_images_and_video.html#videocapture-get: <<CV_CAP_PROP_POS_FRAMES 0-based index of the frame to be decoded/captured next.>> """ capture.set(cv2.CAP_PROP_POS_FRAMES, \ index) common.DebugPrint("MyImageReadMSH(): after capture.set()") # This is only for (paranoid) testing purposes: if config.OCV_OLD_PY_BINDINGS: indexCrt = capture.get(cv2.cv.CV_CAP_PROP_POS_FRAMES) else: """ From http://docs.opencv.org/modules/highgui/doc/reading_and_writing_images_and_video.html#videocapture-get: <<CV_CAP_PROP_POS_FRAMES 0-based index of the frame to be decoded/captured next.>> """ indexCrt = capture.get(cv2.CAP_PROP_POS_FRAMES) #assert int(indexCrt) == index; #!!!!TODO: think if OK if int(indexCrt) != index: common.DebugPrint( \ "MyImageReadMSH(): indexCrt != index --> returning black frame") ret = False else: #common.DebugPrint("Alex: frameR = %d" % frameR); #if myIndex > numFramesR: # break; #ret, img = r_path.read(); ret, img = capture.read() #if ret == False: # break; #!!!!TODO: think if well #assert ret == True; if ret == False: common.DebugPrint( "MyImageReadMSH(index=%d): ret == False --> returning None" % index) img = None else: common.DebugPrint("MyImageReadMSH(): img.shape = %s" % str(img.shape)) common.DebugPrint("MyImageReadMSH(): img.dtype = %s" % str(img.dtype)) #!!!!TODO: I suggest to do the gray conversion at reading, not in multi_scale_harris.py if False: # In the Matlab code he reads gray/8bpp JPEGs imgGray = common.ConvertImgToGrayscale(img) if config.VIDEO_FRAME_RESIZE_SCALING_FACTOR != 1: # We resize the image img = Matlab.imresize(img, \ scale=config.VIDEO_FRAME_RESIZE_SCALING_FACTOR) common.DebugPrint("Exiting MyImageReadMSH()") if False: return imgGray return img
def dp3(Vspace, numFramesR, numFramesQ, BOV_flag): #% Dynamic programming for a maximum-vote path in vote-space #% 2010, Georgios Evangelidis <*****@*****.**> print("Entered dp3(): Running dynamic programming...") if common.MY_DEBUG_STDOUT: common.DebugPrint("dp3(): Vspace = %s" % str(Vspace)) #tic #QD=dir([q_path 'multiharlocs*.mat']); #RD=dir([r_path 'multiharlocs*.mat']); #cross=zeros(length(QD),2); #crossref = np.zeros( (len(q_path), 2) ); crossref = np.zeros((numFramesQ, 2)) #sigma_0=1.2; sigma_0 = 1.2 #[r,c,d] = size(Vspace); r, c, d = Vspace.shape #n=[1:d]; %scale levels n = np.array(range(1, d + 1)) #%scale levels #sigma_I=sqrt(1.8).^n*sigma_0; sigma_I = math.sqrt(1.8)**n * sigma_0 w = sigma_I #%w=w*0+1; #w=w/sum(w); w = w / float(w.sum()) if BOV_flag == 1: #w=fliplr(w); w = w[:, ::-1] #% Initialization #D = zeros(r+1, c+1); D = np.zeros((r + 1, c + 1)) #D(1,:) = NaN; D[0, :] = np.nan #D(:,1) = NaN; D[:, 0] = np.nan #D(1,1) = 0; D[0, 0] = 0 #for i=1:d """ We substitute i - 1 with i since arrays start with 0 in Python, not with 1 like in Matlab. """ #for i in range(1, d + 1): for i in range(d): #V_temp=Vspace(:,:,i); V_temp = Vspace[:, :, i] #V_temp(isnan(V_temp))=0; V_temp[np.isnan(V_temp)] = 0 # !!!!TODO: check OK #Vspace(:,:,i)=V_temp; Vspace[:, :, i] = V_temp #VV=zeros(r,c); VV = np.zeros((r, c)) #for j=1:d """ We substitute j - 1 with j since arrays start with 0 in Python, not with 1 like in Matlab. """ #for j in range(1, d + 1): for j in range(d): #VV=VV+Vspace(:,:,j); VV = VV + Vspace[:, :, j] #D(2:end,2:end)=VV; D[1:, 1:] = VV #NEW_DP3_ALEX = True; NEW_DP3_ALEX = False if NEW_DP3_ALEX: # Alex: added cost cost = np.zeros((r + 1, c + 1)) #% for traceback #Tback = zeros(r+1,c+1); Tback = np.zeros((r + 1, c + 1)) #if True: if common.MY_DEBUG_STDOUT: common.DebugPrint("dp3(): printing locally optimum solutions:") # Alex: trying out to find a better solution than dp3() !!!!TODO : more # This solution is basically the one returned by causal() IF we do NOT apply Matlab.filter2() on Vspace for j in range(1, c + 1): maxCol = 0.0 maxPos = -1 for i in range(1, r + 1): assert D[i, j] >= 0.0 if maxCol < D[i, j]: maxCol = D[i, j] maxPos = i # So for query frame j we have a candidate matching ref frame i common.DebugPrint( "dp3(): for query frame %d - candidate frame %d" % (j - 1, maxPos)) common.DebugPrint( "dp3(): for query frame %d we found matching ref frame %d" % (j - 1, maxPos)) # !!!!TODO: make i =0.., j=0.. and substitute i-1 with i, i-2 with i-1, etc #for i = 1:r; for i in range(1, r + 1): #for j = 1:c for j in range(1, c + 1): #if (i>1) && (j>1) if (i > 1) and (j > 1): #dd1 = w(1)*Vspace(i-1,max(1,j-2),1); dd1 = w[0] * Vspace[i - 2, max(0, j - 3), 0] #dd2 = w(1)*Vspace(max(1,i-2),j-1,1); dd2 = w[0] * Vspace[max(0, i - 3), j - 2, 0] #dd3 = w(1)*Vspace(i-1,j-1,1); dd3 = w[0] * Vspace[i - 2, j - 2, 0] #dd4 = w(1)*Vspace(i-1,j,1); dd4 = w[0] * Vspace[i - 2, j - 1, 0] #dd5 = w(1)*Vspace(i,j-1,1); dd5 = w[0] * Vspace[i - 1, j - 2, 0] if d > 1: #for sc = 2:d for sc in range(2, d + 1): #dd1 = max(dd1, w(sc)*Vspace(i-1,max(1,j-2),sc)); dd1 = max(dd1, w[sc - 1] * \ Vspace[i - 2, max(0, j - 3), sc - 1]) #dd2 = max(dd2, w(sc)*Vspace(max(1,i-2),j-1,sc)); dd2 = max(dd2, w[sc - 1] * \ Vspace[max(0, i - 3), j - 2, sc - 1]) #dd3 = max(dd3, w(sc)*Vspace(i-1,j-1,sc)); dd3 = max(dd3, w[sc - 1] * Vspace[i - 2, j - 2, sc - 1]) #dd4 = max(dd4, w(sc)*Vspace(i-1,j,sc)); dd4 = max(dd4, w[sc - 1] * Vspace[i - 2, j - 1, sc - 1]) #dd5 = max(dd5, w(sc)*Vspace(i,j-1,sc)); dd5 = max(dd5, w[sc - 1] * Vspace[i - 1, j - 2, sc - 1]) #D(i,j-1)=D(i,j-1)+dd1; D[i - 1, j - 2] += dd1 #D(i-1,j)=D(i-1,j)+dd2; D[i - 2, j - 1] += dd2 #D(i,j)=D(i,j)+ dd3; D[i - 1, j - 1] += dd3 #D(i,j+1)=D(i,j+1)+ dd4; D[i - 1, j] += dd4 #D(i+1,j)=D(i+1,j)+ dd5; D[i, j - 1] += dd5 #% % instead of above loop, use the following five lines when scales=6 #% D(i,j-1)=D(i,j-1)+max([w(1)*Vspace(i-1,max(1,j-2),1),w(2)*Vspace(i-1,max(1,j-2),2),w(3)*Vspace(i-1,max(1,j-2),3),w(4)*Vspace(i-1,max(1,j-2),4),w(5)*Vspace(i-1,max(1,j-2),5),w(6)*Vspace(i-1,max(1,j-2),6)]); #% D(i-1,j)=D(i-1,j)+max([w(1)*Vspace(max(1,i-2),j-1,1),w(2)*Vspace(max(1,i-2),j-1,2),w(3)*Vspace(max(1,i-2),j-1,3),w(4)*Vspace(max(1,i-2),j-1,4),w(5)*Vspace(max(1,i-2),j-1,5),w(6)*Vspace(max(1,i-2),j-1,6)]); #% D(i,j)=D(i,j)+max([w(1)*Vspace(i-1,j-1,1),w(2)*Vspace(i-1,j-1,2),w(3)*Vspace(i-1,j-1,3),w(4)*Vspace(i-1,j-1,4),w(5)*Vspace(i-1,j-1,5),w(6)*Vspace(i-1,j-1,6)]); #% D(i,j+1)=D(i,j+1)+max([w(1)*Vspace(i-1,j,1),w(2)*Vspace(i-1,j,2),w(3)*Vspace(i-1,j,3),w(4)*Vspace(i-1,j,4),w(5)*Vspace(i-1,j,5),w(6)*Vspace(i-1,j,6)]); #% D(i+1,j)=D(i+1,j)+max([w(1)*Vspace(i,j-1,1),w(2)*Vspace(i,j-1,2),w(3)*Vspace(i,j-1,3),w(4)*Vspace(i,j-1,4),w(5)*Vspace(i,j-1,5),w(6)*Vspace(i,j-1,6)]); #% [dmax, tb] = max([D(i, j), D(i-1, j), D(i, j-1), D(i,j+1),D(i+1,j)]); #[dmax, tb] = max([D(i, j)+1/sqrt(2), D(i-1, j)+1/sqrt(5), D(i, j-1)+1/sqrt(5), D(i,j+1)+1, D(i+1,j)+1]); dmax, tb = Matlab.max(np.array([ \ D[i - 1, j - 1] + 1.0 / math.sqrt(2.0), \ D[i - 2, j - 1] + 1.0 / math.sqrt(5.0), \ D[i - 1, j - 2] + 1.0 / math.sqrt(5.0), \ D[i - 1, j] + 1, D[i, j - 1] + 1])) else: #[dmax, tb] = max([D(i,j), D(i,j+1), D(i+1,j)]); dmax, tb = Matlab.max( \ np.array([D[i - 1, j - 1], D[i - 1, j], D[i, j - 1]])) if common.MY_DEBUG_STDOUT: common.DebugPrint("dp3(): dmax = %s" % str(dmax)) common.DebugPrint("dp3(): tb = %s" % str(tb)) #D(i+1,j+1) = D(i+1,j+1)+dmax; if NEW_DP3_ALEX: cost[i, j] = 0 #!!!!TODO: think more else: D[i, j] += dmax #!!!!TODO: for me it's weird he adds dmax here... #Tback(i+1,j+1) = tb; Tback[i, j] = tb if common.MY_DEBUG_STDOUT: common.DebugPrint("dp3(): D.shape = %s" % str(D.shape)) common.DebugPrint("dp3(): D = %s" % str(D)) common.DebugPrint("dp3(): Tback.shape = %s" % str(Tback.shape)) common.DebugPrint("dp3(): Tback = %s" % str(Tback)) #% Traceback i = r + 1 j = c + 1 y = i - 1 x = j - 1 #while i > 2 & j > 2 while (i > 2) and (j > 2): #tb = Tback(i,j); tb = Tback[i - 1, j - 1] + 1 # In Matlab, max returns indices from 1.. if tb == 1: i -= 1 j -= 1 elif tb == 2: i -= 2 j -= 1 elif tb == 3: i -= 1 j -= 2 elif tb == 4: i -= 1 j = j elif tb == 5: j -= 1 i = i else: #error; assert False #y = [i,y]; #NOT GOOD: y = np.c_[i, y]; y = np.hstack([i - 1, y]) #x = [j,x]; #NOT GOOD: x = np.c_[j, x]; x = np.hstack([j - 1, x]) if common.MY_DEBUG_STDOUT: common.DebugPrint("dp3(): before D.shape = %s" % str(D.shape)) common.DebugPrint("dp3(): before D = %s" % str(D)) #% Strip off the edges of the D matrix before returning #D = D(2:(r+1),2:(c+1)); D = D[1:(r + 1), 1:(c + 1)] if common.MY_DEBUG_STDOUT: common.DebugPrint("dp3(): D.shape = %s" % str(D.shape)) common.DebugPrint("dp3(): D = %s" % str(D)) #RD_start=str2num(RD(1).name(end-9:end-4)); RD_start = 1 if common.MY_DEBUG_STDOUT: common.DebugPrint("dp3(): Vspace.shape = %s" % str(Vspace.shape)) #!!!!TODO: understand well what is x,y and why computes p #for i=1:size(Vspace,2): for i in range(0, Vspace.shape[1]): #cross(i,1)=str2num(QD(i).name(end-9:end-4)); crossref[i, 0] = i #i; #!!!!TODO: think if OK #p=find(x==i); p = np.nonzero(x == i) p = p[0] if common.MY_DEBUG_STDOUT: common.DebugPrint("dp3(): x.shape = %s" % str(x.shape)) common.DebugPrint("dp3(): x = %s" % str(x)) common.DebugPrint("dp3(): y.shape = %s" % str(y.shape)) common.DebugPrint("dp3(): y = %s" % str(y)) common.DebugPrint("dp3(): i = %s" % str(i)) common.DebugPrint("dp3(): p = %s" % str(p)) #if isempty(p) if p.size == 0: #% Alex: Vali Codreanu said to change from temp=0; to temp=3; #temp=3; temp = 0 if common.MY_DEBUG_STDOUT: common.DebugPrint("dp3(): temp = %s" % str(temp)) crossref[i, 1] = 0 + RD_start - 1 else: #temp=y(p); temp = y[p] if common.MY_DEBUG_STDOUT: common.DebugPrint("dp3(): temp = %s" % str(temp)) #cross(i,2)=temp(end)+RD_start-1; if temp.size == 1: # If temp has only 1 element: crossref[i, 1] = temp + RD_start - 1 else: crossref[i, 1] = temp[-1] + RD_start - 1 #assert temp.size == 1; #common.DebugPrint("dp3(): temp = %s" % str(temp)); """ #cross(i,2)=temp(end)+RD_start-1; if True: crossref[i, 1] = temp[-1] + RD_start - 1; else: # BAD IDEA!!!!TODO crossref[i, 1] = temp + RD_start - 1; """ #printf("dp3(): Done in blabla secs\n"); common.DebugPrint("dp3(): crossref = %s" % str(crossref)) ComputeCost(crossref, VV, "crossref_dp3.txt") #return [y,x,D,Tback,cross] return y, x, D, Tback, crossref
def causal(Vspace, H, numFramesQ, numFramesR, BOV_flag, cropflag, const_type, nop=0): #%cons_type : 1 means that spatial constraint applies after voting #% 2 means that spatial constraint applies before voting -> NOT #% USED IN PAPER # causal() is the local/greedy optimization solution #% nargin>7 means that we do the local smoothing with RANSAC (see the paper) """ common.DebugPrint("causal(): At entrance Vspace=%s,\nH=%s, \n" \ "numFramesQ=%s, numFramesR=%s, BOV_flag=%d, cropflag=%d, const_type=%d, " \ "nop=%d" % \ (str(Vspace), str(H), numFramesQ, numFramesR, BOV_flag, cropflag, \ const_type, nop)); """ print ( "causal(): At entrance \n" " numFramesQ=%s, numFramesR=%s, BOV_flag=%d, cropflag=%d, const_type=%d, " " nop=%d" % (numFramesQ, numFramesR, BOV_flag, cropflag, const_type, nop) ) # Normally BOV_flag=0, cropflag=0, const_type=1,nop=0 # if True: if common.MY_DEBUG_STDOUT: print ("causal(): Vspace.shape = %s" % str(Vspace.shape)) print ("causal(): H.shape = %s" % str(H.shape)) for i in range(Vspace.shape[2]): common.DebugPrint("causal(): Vspace[:, :, %d] = \n%s" % (i, str(Vspace[:, :, i]))) """ common.DebugPrint("causal(): Vspace[:, :, %d] = " % (i)); for r in range(Vspace.shape[0]): for c in range(Vspace.shape[1]): common.DebugPrint("%.4f " % Vspace[r, c, i]), print; """ print for i in range(H.shape[2]): common.DebugPrint("causal(): H[:, :, %d] = \n%s" % (i, str(H[:, :, i]))) """ common.DebugPrint("causal(): H[:, :, %d] = " % (i)); for r in range(H.shape[0]): for c in range(H.shape[1]): common.DebugPrint("%.4f " % H[r, c, i]), print; """ # nos=size(Vspace,3); %3D matrix nos = Vspace.shape[2] #%3D matrix #% same with the multi_scale_harris function sigma_0 = 1.2 # n=0:nos-1; n = range(0, nos) # sigma_D=sqrt(1.8).^n*sigma_0; % We should see sqrt(1.8) returning a vector of nos elements # NOT GOOD in Python: sigma_D = math.sqrt(1.8)**n * sigma_0; sq18 = np.ones((1, nos)) * math.sqrt(1.8) sigma_D = sq18 ** n * sigma_0 # w = sigma_D; w = sigma_D[0] # common.DebugPrint("causal(): w = %s" % str(w)); # common.DebugPrint("causal(): w.shape = %s" % str(w.shape)); w = w / w.sum() # common.DebugPrint("causal(): w = %s" % str(w)); # Alex: normally NOT executed if BOV_flag == 1: # w=fliplr(w); w = w[:, ::-1] # We flip on the vertical (left becomes right) print ("causal(): w = %s" % str(w)) # Alex: normally NOT executed if (const_type == 1) and (BOV_flag == 1): assert False # Normally this code does NOT get executed # VV=zeros(size(Vspace,1),size(Vspace,2)); VV = np.zeros((Vspace.shape[0], Vspace.shape[1])) for j in range(1, nos + 1): # VV=VV+Vspace(:,:,j); VV = VV + Vspace[:, :, j - 1] # [X,Y]=sort(VV,'descend'); X, Y = sort(VV, "descend") #%enable top-N list #%N = 50; #%N = 100; N = 300 for s in range(1, nos + 1): for i in range(1, Vspace.shape[1] + 1): # y=Y(:,i); y = Y[:, i - 1] # votes=Vspace(:,i,s); votes = Vspace[:, i - 1, s - 1] # h=H(:,i,s); h = H[:, i - 1, s - 1] # votes(y(N+1:end))=0; votes[y[N:]] = 0 # h(y(N+1:end))=0; h[y[N:]] = 0 # Vspace(:,i,s)=votes; Vspace[:, i - 1, s - 1] = votes # H(:,i,s)=h; H[:, i - 1, s - 1] = h # clear VV VV = None # QD=dir([q_path 'multiharlocs*.mat']); # RD=dir([r_path 'multiharlocs*.mat']); # QD = [None] * numFramesQ; # RD = [None] * numFramesR; # cross=zeros(length(QD),2); # crossref = np.zeros( (len(QD), 2) ); crossref = np.zeros((numFramesQ, 2)) # common.DebugPrint("causal(): crossref = %s" % str(crossref)); #% str=['load ' r_path 'multiscale_nod']; #% eval(str) #%replace nan votes with zeros #%sum(sum(isnan(Vspace(:,:,1)))) # We transform nan in 0 in Vspace """ We substitute i - 1 with i, since array indexing starts from 1 in Matlab and 0 in Python. """ # for i in range(1, nos + 1): for i in range(nos): # V_temp=Vspace(:,:,i); # V_temp = Vspace[:, :, i - 1]; V_temp = Vspace[:, :, i] # V_temp(isnan(V_temp))=0; V_temp[np.isnan(V_temp)] = 0 # Vspace(:,:,i)=V_temp; # Vspace[:, :, i - 1] = V_temp; Vspace[:, :, i] = V_temp # Alex: I personally find this idea of using filter2() VERY BAD - the results in Vspace should already be VERY good if CAUSAL_DO_NOT_SMOOTH == False: #% this filtering of votes favors smoother results #%b=ones(11,1); #%b=b/prod(size(b)); b = Matlab.hamming(11) #% USED # if False: """ We substitute i - 1 with i, since array indexing starts from 1 in Matlab and 0 in Python. """ # for i in range(1, nos + 1): for i in range(nos): # Vspace(:,:,i)=filter2(b,Vspace(:,:,i)); """ From the Matlab help: Y = filter2(h,X) filters the data in X with the two-dimensional FIR filter in the matrix h. It computes the result, Y, using two-dimensional correlation, and returns the central part of the correlation that is the same size as X. """ # Vspace[:, :, i - 1] = Matlab.filter2(b, Vspace[:, :, i - 1]); Vspace[:, :, i] = Matlab.filter2(b, Vspace[:, :, i]) #% use imfilter if filter2 takes time #!!!!TODO: do the optimization Evangelidis says here """ From Matlab help: M = mean(A,dim) returns the mean values for elements along the dimension of A specified by scalar dim. For matrices, mean(A,2) is a column vector containing the mean value of each row. """ # V=mean(Vspace,3); % this might help more instead of starting from zero votes V = Vspace.mean(2) # this might help more instead of starting from zero votes # common.DebugPrint("causal(): V.shape = %s" % str(V.shape)); """ We substitute i - 1 with i, since array indexing starts from 1 in Matlab and 0 in Python. """ # for i in range(1, nos + 1): for i in range(nos): if cropflag == 0: if (const_type == 1) and (BOV_flag == 1): # V=V+w(i)*H(:,:,i)+Vspace(:,:,i); # V += w[i - 1] * H[:, :, i - 1] + Vspace[:, :, i - 1]; #!!!!TODO: think well * V += w[i] * H[:, :, i] + Vspace[:, :, i] #!!!!TODO: think well * else: # Alex: we are normally in this case, since cropflag == 0, const_type == 1, BOV_flag == 0 # V=V+w(i)*Vspace(:,:,i); # V += w[i - 1] * Vspace[:, :, i - 1]; #!!!!TODO: think well * Exception "ValueError: operands could not be broadcast together with shapes (5) (23,8)" V += w[i] * Vspace[:, :, i] #!!!!TODO: think well * Exception "ValueError: operands could not be broadcast together with shapes (5) (23,8)" else: # V=V+w(i)*Vspace(:,:,i)+H(:,:,i); # V += w[i - 1] * Vspace[:, :, i - 1] + H[:, :, i - 1]; #!!!!TODO: think well * V += w[i] * Vspace[:, :, i] + H[:, :, i] #!!!!TODO: think well * if common.MY_DEBUG_STDOUT: # common.DebugPrint("causal(): Vspace.shape = %s" % str(Vspace.shape)); common.DebugPrint("causal(): V.shape = %s" % str(V.shape)) common.DebugPrint("causal(): V (the matrix used to choose the max-voting reference frame) = %s" % str(V)) # common.DebugPrint("causal(): len(QD) = %d" % len(QD)); # for iFor in range(1, len(QD) + 1): """ We substitute iFor -1 with iFor since arrays start with 0 in Python, not with 1 like in Matlab """ # for iFor in range(1, numFramesQ + 1): for iFor in range(numFramesQ): # cross(i,1)=str2num(QD(i).name(end-9:end-4)); crossref[iFor, 0] = iFor # TODO - think well # [a,b]=max(V(:,i)); b = V[:, iFor].argmax() a = V[:, iFor][b] # cross(i,2)=str2num(RD(b).name(end-9:end-4)); crossref[iFor, 1] = b #!!!!TODO - think well # We normally do NOT execute the following code, since nop == 0 # if nargin>7 if nop != 0: # XX = cross(:,1)'; XX = crossref[:, 0].T # YY = cross(:,2)'; YY = crossref[:, 1].T YY_new = YY if mod(nop, 2) == 0: # error('Number of points (parameter NOP) must be odd number'); # common.DebugPrint("Number of points (parameter NOP) must be odd number"); quit() # miso is used only as array index and in range expressions --> it can be an integer miso = (nop - 1) / 2 if nop < 7: # common.DebugPrint("Choose more than 5 points for local fitting"); pass # for i = miso+1:length(XX)-miso for iFor in range(miso + 1, len(XX) - miso + 1): # xx = XX(i-miso:i+miso); xx = XX[iFor - miso - 1 : iFor + miso] # yy = YY(i-miso:i+miso); yy = YY[iFor - miso - 1 : iFor + miso] if nop < 9: """ iterNum is used only in range expressions and array dimensions --> it can be an integer. """ iterNum = nop * (nop - 1) / 2 else: iterNum = 10 thDist = 2 thInlrRatio = 0.6 """ From Matlab help: RANSAC Use RANdom SAmple Consensus to fit a line RESCOEF = RANSAC(PTS,ITERNUM,THDIST,THINLRRATIO) PTS is 2*n matrix including n points, ITERNUM is the number of iteration, THDIST is the inlier distance threshold and ROUND(THINLRRATIO*SIZE(PTS,2)) is the inlier number threshold. The final fitted line is y = alpha*x+beta. Yan Ke @ THUEE, [email protected] """ # [ alpha, beta ] = ransac_line( [xx;yy],iterNum,thDist,thInlrRatio ); alpha, beta = ransac_line(np.r_[xx, yy], iterNum, thDist, thInlrRatio) if alpha != 0: yy_new = alpha * xx + beta #!!!!TODO: think well * YY_new[iFor - 1] = yy_new[miso + 1 - 1] # cross(:,2) = YY_new; crossref[:, 2] = YY_new crossref = crossref.astype(int) print ("causal(): crossref = %s" % str(crossref)) ComputeCost(crossref, V, "crossref_causal.txt") if False: # True: # print("Am here1111"); # causal_Alex(Vspace, numFramesQ, numFramesR); # print("Am here2222"); # dp3(Vspace, numFramesR, numFramesQ, BOV_flag); # print("Am here3333"); # dp_Alex(Vspace, numFramesR, numFramesQ, BOV_flag); dp_Alex(Vspace, numFramesR, numFramesQ, BOV_flag, PREV_REF=5, NEXT_REF=-1) return crossref
def causal(Vspace, H, numFramesQ, numFramesR, BOV_flag, cropflag, const_type, nop=0): #%cons_type : 1 means that spatial constraint applies after voting #% 2 means that spatial constraint applies before voting -> NOT #% USED IN PAPER # causal() is the local/greedy optimization solution #% nargin>7 means that we do the local smoothing with RANSAC (see the paper) """ common.DebugPrint("causal(): At entrance Vspace=%s,\nH=%s, \n" \ "numFramesQ=%s, numFramesR=%s, BOV_flag=%d, cropflag=%d, const_type=%d, " \ "nop=%d" % \ (str(Vspace), str(H), numFramesQ, numFramesR, BOV_flag, cropflag, \ const_type, nop)); """ print("causal(): At entrance \n" \ " numFramesQ=%s, numFramesR=%s, BOV_flag=%d, cropflag=%d, const_type=%d, " \ " nop=%d" % \ (numFramesQ, numFramesR, BOV_flag, cropflag, \ const_type, nop)) # Normally BOV_flag=0, cropflag=0, const_type=1,nop=0 #if True: if common.MY_DEBUG_STDOUT: print("causal(): Vspace.shape = %s" % str(Vspace.shape)) print("causal(): H.shape = %s" % str(H.shape)) for i in range(Vspace.shape[2]): common.DebugPrint("causal(): Vspace[:, :, %d] = \n%s" % (i, str(Vspace[:, :, i]))) """ common.DebugPrint("causal(): Vspace[:, :, %d] = " % (i)); for r in range(Vspace.shape[0]): for c in range(Vspace.shape[1]): common.DebugPrint("%.4f " % Vspace[r, c, i]), print; """ print for i in range(H.shape[2]): common.DebugPrint("causal(): H[:, :, %d] = \n%s" % (i, str(H[:, :, i]))) """ common.DebugPrint("causal(): H[:, :, %d] = " % (i)); for r in range(H.shape[0]): for c in range(H.shape[1]): common.DebugPrint("%.4f " % H[r, c, i]), print; """ #nos=size(Vspace,3); %3D matrix nos = Vspace.shape[2] #%3D matrix #% same with the multi_scale_harris function sigma_0 = 1.2 #n=0:nos-1; n = range(0, nos) #sigma_D=sqrt(1.8).^n*sigma_0; % We should see sqrt(1.8) returning a vector of nos elements #NOT GOOD in Python: sigma_D = math.sqrt(1.8)**n * sigma_0; sq18 = np.ones((1, nos)) * math.sqrt(1.8) sigma_D = sq18**n * sigma_0 #w = sigma_D; w = sigma_D[0] #common.DebugPrint("causal(): w = %s" % str(w)); #common.DebugPrint("causal(): w.shape = %s" % str(w.shape)); w = w / w.sum() #common.DebugPrint("causal(): w = %s" % str(w)); # Alex: normally NOT executed if BOV_flag == 1: #w=fliplr(w); w = w[:, ::-1] # We flip on the vertical (left becomes right) print("causal(): w = %s" % str(w)) # Alex: normally NOT executed if (const_type == 1) and (BOV_flag == 1): assert False # Normally this code does NOT get executed #VV=zeros(size(Vspace,1),size(Vspace,2)); VV = np.zeros((Vspace.shape[0], Vspace.shape[1])) for j in range(1, nos + 1): #VV=VV+Vspace(:,:,j); VV = VV + Vspace[:, :, j - 1] #[X,Y]=sort(VV,'descend'); X, Y = sort(VV, "descend") #%enable top-N list #%N = 50; #%N = 100; N = 300 for s in range(1, nos + 1): for i in range(1, Vspace.shape[1] + 1): #y=Y(:,i); y = Y[:, i - 1] #votes=Vspace(:,i,s); votes = Vspace[:, i - 1, s - 1] #h=H(:,i,s); h = H[:, i - 1, s - 1] #votes(y(N+1:end))=0; votes[y[N:]] = 0 #h(y(N+1:end))=0; h[y[N:]] = 0 #Vspace(:,i,s)=votes; Vspace[:, i - 1, s - 1] = votes #H(:,i,s)=h; H[:, i - 1, s - 1] = h #clear VV VV = None #QD=dir([q_path 'multiharlocs*.mat']); #RD=dir([r_path 'multiharlocs*.mat']); #QD = [None] * numFramesQ; #RD = [None] * numFramesR; #cross=zeros(length(QD),2); #crossref = np.zeros( (len(QD), 2) ); crossref = np.zeros((numFramesQ, 2)) #common.DebugPrint("causal(): crossref = %s" % str(crossref)); #% str=['load ' r_path 'multiscale_nod']; #% eval(str) #%replace nan votes with zeros #%sum(sum(isnan(Vspace(:,:,1)))) # We transform nan in 0 in Vspace """ We substitute i - 1 with i, since array indexing starts from 1 in Matlab and 0 in Python. """ #for i in range(1, nos + 1): for i in range(nos): #V_temp=Vspace(:,:,i); #V_temp = Vspace[:, :, i - 1]; V_temp = Vspace[:, :, i] #V_temp(isnan(V_temp))=0; V_temp[np.isnan(V_temp)] = 0 #Vspace(:,:,i)=V_temp; #Vspace[:, :, i - 1] = V_temp; Vspace[:, :, i] = V_temp # Alex: I personally find this idea of using filter2() VERY BAD - the results in Vspace should already be VERY good if CAUSAL_DO_NOT_SMOOTH == False: #% this filtering of votes favors smoother results #%b=ones(11,1); #%b=b/prod(size(b)); b = Matlab.hamming(11) #% USED #if False: """ We substitute i - 1 with i, since array indexing starts from 1 in Matlab and 0 in Python. """ #for i in range(1, nos + 1): for i in range(nos): #Vspace(:,:,i)=filter2(b,Vspace(:,:,i)); """ From the Matlab help: Y = filter2(h,X) filters the data in X with the two-dimensional FIR filter in the matrix h. It computes the result, Y, using two-dimensional correlation, and returns the central part of the correlation that is the same size as X. """ #Vspace[:, :, i - 1] = Matlab.filter2(b, Vspace[:, :, i - 1]); Vspace[:, :, i] = Matlab.filter2(b, Vspace[:, :, i]) #% use imfilter if filter2 takes time #!!!!TODO: do the optimization Evangelidis says here """ From Matlab help: M = mean(A,dim) returns the mean values for elements along the dimension of A specified by scalar dim. For matrices, mean(A,2) is a column vector containing the mean value of each row. """ #V=mean(Vspace,3); % this might help more instead of starting from zero votes V = Vspace.mean( 2) # this might help more instead of starting from zero votes #common.DebugPrint("causal(): V.shape = %s" % str(V.shape)); """ We substitute i - 1 with i, since array indexing starts from 1 in Matlab and 0 in Python. """ #for i in range(1, nos + 1): for i in range(nos): if cropflag == 0: if (const_type == 1) and (BOV_flag == 1): #V=V+w(i)*H(:,:,i)+Vspace(:,:,i); #V += w[i - 1] * H[:, :, i - 1] + Vspace[:, :, i - 1]; #!!!!TODO: think well * V += w[i] * H[:, :, i] + Vspace[:, :, i] #!!!!TODO: think well * else: # Alex: we are normally in this case, since cropflag == 0, const_type == 1, BOV_flag == 0 #V=V+w(i)*Vspace(:,:,i); #V += w[i - 1] * Vspace[:, :, i - 1]; #!!!!TODO: think well * Exception "ValueError: operands could not be broadcast together with shapes (5) (23,8)" V += w[i] * Vspace[:, :, i] #!!!!TODO: think well * Exception "ValueError: operands could not be broadcast together with shapes (5) (23,8)" else: #V=V+w(i)*Vspace(:,:,i)+H(:,:,i); #V += w[i - 1] * Vspace[:, :, i - 1] + H[:, :, i - 1]; #!!!!TODO: think well * V += w[i] * Vspace[:, :, i] + H[:, :, i] #!!!!TODO: think well * if common.MY_DEBUG_STDOUT: #common.DebugPrint("causal(): Vspace.shape = %s" % str(Vspace.shape)); common.DebugPrint("causal(): V.shape = %s" % str(V.shape)) common.DebugPrint( "causal(): V (the matrix used to choose the max-voting reference frame) = %s" % str(V)) #common.DebugPrint("causal(): len(QD) = %d" % len(QD)); #for iFor in range(1, len(QD) + 1): """ We substitute iFor -1 with iFor since arrays start with 0 in Python, not with 1 like in Matlab """ #for iFor in range(1, numFramesQ + 1): for iFor in range(numFramesQ): #cross(i,1)=str2num(QD(i).name(end-9:end-4)); crossref[iFor, 0] = iFor #TODO - think well #[a,b]=max(V(:,i)); b = V[:, iFor].argmax() a = V[:, iFor][b] #cross(i,2)=str2num(RD(b).name(end-9:end-4)); crossref[iFor, 1] = b #!!!!TODO - think well # We normally do NOT execute the following code, since nop == 0 #if nargin>7 if nop != 0: #XX = cross(:,1)'; XX = crossref[:, 0].T #YY = cross(:,2)'; YY = crossref[:, 1].T YY_new = YY if mod(nop, 2) == 0: #error('Number of points (parameter NOP) must be odd number'); #common.DebugPrint("Number of points (parameter NOP) must be odd number"); quit() # miso is used only as array index and in range expressions --> it can be an integer miso = (nop - 1) / 2 if nop < 7: #common.DebugPrint("Choose more than 5 points for local fitting"); pass #for i = miso+1:length(XX)-miso for iFor in range(miso + 1, len(XX) - miso + 1): #xx = XX(i-miso:i+miso); xx = XX[iFor - miso - 1:iFor + miso] #yy = YY(i-miso:i+miso); yy = YY[iFor - miso - 1:iFor + miso] if nop < 9: """ iterNum is used only in range expressions and array dimensions --> it can be an integer. """ iterNum = nop * (nop - 1) / 2 else: iterNum = 10 thDist = 2 thInlrRatio = 0.6 """ From Matlab help: RANSAC Use RANdom SAmple Consensus to fit a line RESCOEF = RANSAC(PTS,ITERNUM,THDIST,THINLRRATIO) PTS is 2*n matrix including n points, ITERNUM is the number of iteration, THDIST is the inlier distance threshold and ROUND(THINLRRATIO*SIZE(PTS,2)) is the inlier number threshold. The final fitted line is y = alpha*x+beta. Yan Ke @ THUEE, [email protected] """ #[ alpha, beta ] = ransac_line( [xx;yy],iterNum,thDist,thInlrRatio ); alpha, beta = ransac_line( np.r_[xx, yy], iterNum, \ thDist, thInlrRatio ) if alpha != 0: yy_new = alpha * xx + beta #!!!!TODO: think well * YY_new[iFor - 1] = yy_new[miso + 1 - 1] #cross(:,2) = YY_new; crossref[:, 2] = YY_new crossref = crossref.astype(int) print("causal(): crossref = %s" % str(crossref)) ComputeCost(crossref, V, "crossref_causal.txt") if False: #True: #print("Am here1111"); #causal_Alex(Vspace, numFramesQ, numFramesR); #print("Am here2222"); #dp3(Vspace, numFramesR, numFramesQ, BOV_flag); #print("Am here3333"); #dp_Alex(Vspace, numFramesR, numFramesQ, BOV_flag); dp_Alex(Vspace, numFramesR, numFramesQ, BOV_flag, PREV_REF=5, NEXT_REF=-1) return crossref
def MyImageReadMSH(index): global g; """ common.DebugPrint("MyImageReadMSH(): initFrame = %d)" % \ (config.initFrame[g.indexVideo])); index += config.initFrame[g.indexVideo]; """ """ common.DebugPrint("Entered MyImageReadMSH(capture=%s, index=%s)" % \ (str(capture), str(index))); """ common.DebugPrint("Entered MyImageReadMSH(index=%s)" % \ (str(index))); """ We must reopen the capture device in each different process, otherwise the program blocks at the first operation in the "global" capture device. """ if g.harlocsFolder.endswith(config.HARRIS_QUERY_FOLDER_NAME): if g.captureQ == None: capture = cv2.VideoCapture(sys.argv[1]); g.captureQ = capture; common.DebugPrint("MyImageReadMSH(): new capture=%s" % \ (str(capture))); else: capture = g.captureQ; elif g.harlocsFolder.endswith(config.HARRIS_REFERENCE_FOLDER_NAME): if g.captureR == None: capture = cv2.VideoCapture(sys.argv[2]); g.captureR = capture; common.DebugPrint("MyImageReadMSH(): new capture=%s" % \ (str(capture))); else: capture = g.captureR; else: assert False; assert (g.indexVideo == 0) or (g.indexVideo == 1); if config.OCV_OLD_PY_BINDINGS: capture.set(cv2.cv.CV_CAP_PROP_POS_FRAMES, \ index); else: """ From http://docs.opencv.org/modules/highgui/doc/reading_and_writing_images_and_video.html#videocapture-get: <<CV_CAP_PROP_POS_FRAMES 0-based index of the frame to be decoded/captured next.>> """ capture.set(cv2.CAP_PROP_POS_FRAMES, \ index); common.DebugPrint("MyImageReadMSH(): after capture.set()"); # This is only for (paranoid) testing purposes: if config.OCV_OLD_PY_BINDINGS: indexCrt = capture.get(cv2.cv.CV_CAP_PROP_POS_FRAMES); else: """ From http://docs.opencv.org/modules/highgui/doc/reading_and_writing_images_and_video.html#videocapture-get: <<CV_CAP_PROP_POS_FRAMES 0-based index of the frame to be decoded/captured next.>> """ indexCrt = capture.get(cv2.CAP_PROP_POS_FRAMES); #assert int(indexCrt) == index; #!!!!TODO: think if OK if int(indexCrt) != index: common.DebugPrint( \ "MyImageReadMSH(): indexCrt != index --> returning black frame"); ret = False; else: #common.DebugPrint("Alex: frameR = %d" % frameR); #if myIndex > numFramesR: # break; #ret, img = r_path.read(); ret, img = capture.read(); #if ret == False: # break; #!!!!TODO: think if well #assert ret == True; if ret == False: common.DebugPrint( "MyImageReadMSH(index=%d): ret == False --> returning None" % index); img = None; else: common.DebugPrint("MyImageReadMSH(): img.shape = %s" % str(img.shape)); common.DebugPrint("MyImageReadMSH(): img.dtype = %s" % str(img.dtype)); #!!!!TODO: I suggest to do the gray conversion at reading, not in multi_scale_harris.py if False: # In the Matlab code he reads gray/8bpp JPEGs imgGray = common.ConvertImgToGrayscale(img); if config.VIDEO_FRAME_RESIZE_SCALING_FACTOR != 1: # We resize the image img = Matlab.imresize(img, \ scale=config.VIDEO_FRAME_RESIZE_SCALING_FACTOR); common.DebugPrint("Exiting MyImageReadMSH()"); if False: return imgGray; return img;
def ComputeHarlocs(capture, counterStep, folderName, fileNamePrefix, fileNameExtension=".csv", indexVideo=-1): print( \ "Entered ComputeHarlocs(capture=%s, counterStep=%d, folderName=%s, " \ "indexVideo=%d)" % \ (str(capture), counterStep, folderName, indexVideo)); harlocsFolder = config.VIDEOS_FOLDER + "/" + folderName; t1 = float(cv2.getTickCount()); harlocs = []; if config.OCV_OLD_PY_BINDINGS: numFrames = int(capture.get(cv2.cv.CV_CAP_PROP_FRAME_COUNT)); else: numFrames = int(capture.get(cv2.CAP_PROP_FRAME_COUNT)); common.DebugPrint("ComputeHarlocs(): numFrames = %d" % numFrames); if not os.path.exists(harlocsFolder): os.makedirs(harlocsFolder); else: #!!!!TODO: check that the loaded Harlocs are complete - same frame numbers as in the videos # Folder with precomputed Harris features exists folderContent = os.listdir(harlocsFolder); sortedFolderContent = sorted(folderContent); for fileName in sortedFolderContent: pathFileName = harlocsFolder + "/" + fileName; """ common.DebugPrint("ComputeHarlocs(): pathFileName = %s" % pathFileName); common.DebugPrint("ComputeHarlocs(): fileName = %s" % fileName); """ if os.path.isfile(pathFileName) and \ fileName.startswith(fileNamePrefix) and \ pathFileName.endswith(fileNameExtension): common.DebugPrint("ComputeHarlocs(): Loading %s" % pathFileName); harrisFeatures = multi_scale_harris.LoadMultiScaleHarrisFeatures(pathFileName); harlocs.append(harrisFeatures); if config.endFrame[indexVideo] == -1: assert (len(harlocs) + config.initFrame[indexVideo]) == numFrames; #!!!!TODO: if condition is NOT met, give a nicer error, or redo computations of Harlocs else: assert (len(harlocs) + config.initFrame[indexVideo]) == config.endFrame[indexVideo] + 1; #!!!!TODO: if condition is NOT met, give a nicer error, or redo computations of Harlocs return harlocs; if config.USE_MULTITHREADING == True: global g; g.captureQ = None; # We need to reopen the capture device in each process, separately g.captureR = None; # We need to reopen the capture device in each process, separately #g.capture = capture; g.harlocsFolder = harlocsFolder; g.fileNamePrefix = fileNamePrefix; g.fileNameExtension = fileNameExtension; g.indexVideo = indexVideo; if config.OCV_OLD_PY_BINDINGS: frameCount = int(capture.get(cv2.cv.CV_CAP_PROP_FRAME_COUNT)); else: frameCount = int(capture.get(cv2.CAP_PROP_FRAME_COUNT)); #listParams = range(frameCount); listParams = range(config.initFrame[indexVideo], frameCount, counterStep); common.DebugPrint("ComputeHarlocs(): frameCount = %d" % frameCount); print("ComputeHarlocs(): Spawning a pool of %d workers" % \ config.numProcesses); """ # DEBUG purposes ONLY - since when we use Pool() and call function, if # we have an error in the function the exception reported is very # vague... for i in listParams: IterationStandaloneMSH(i); #import time #time.sleep(1000); """ """ Start worker processes to use on multi-core processor (circumvent also the GIL issue). """ pool = multiprocessing.Pool(processes=config.numProcesses); print("ComputeHarlocs(): Spawned a pool of %d workers" % \ config.numProcesses); #res = pool.map(IterationStandaloneMSH, listParams); # See https://docs.python.org/2/library/multiprocessing.html#module-multiprocessing.pool res = pool.map(func=IterationStandaloneMSH, iterable=listParams, \ chunksize=1); print("Pool.map returns %s" % str(res)); """ From https://medium.com/building-things-on-the-internet/40e9b2b36148 close the pool and wait for the work to finish """ pool.close(); pool.join(); #!!!!TODO: do more efficient - don't load the results from the CSV files return ComputeHarlocs(capture, counterStep, folderName, \ fileNamePrefix, fileNameExtension, indexVideo); #return []; #indexHarloc = 0; while True: if config.OCV_OLD_PY_BINDINGS: framePos = capture.get(cv2.cv.CV_CAP_PROP_POS_FRAMES); else: """ From http://docs.opencv.org/modules/highgui/doc/reading_and_writing_images_and_video.html#videocapture-get: <<CV_CAP_PROP_POS_FRAMES 0-based index of the frame to be decoded/captured next.>> """ framePos = capture.get(cv2.CAP_PROP_POS_FRAMES); common.DebugPrint("ComputeHarlocs(): framePos = %d" % framePos); counter = int(framePos); #0 common.DebugPrint("ComputeHarlocs(): counter = %d" % counter); ret, img = capture.read(); if common.MY_DEBUG_STDOUT: common.DebugPrint("ComputeHarlocs(): img = %s" % str(img)); if False and config.SAVE_FRAMES: fileName = config.IMAGES_FOLDER + "/img_%05d.png" % counter; if not os.path.exists(fileName): #print "dir(img) = %s"% str(dir(img)) """ imgCV = cv.fromarray(img) cv2.imwrite(fileName, imgCV) """ cv2.imwrite(fileName, img); #if ret == False: #MatchFrames.counterQ == 3: if (ret == False) or ((counter > numFrames) or \ (config.endFrame[indexVideo] != -1 and \ counter > config.endFrame[indexVideo])): break; if config.VIDEO_FRAME_RESIZE_SCALING_FACTOR != 1: img = Matlab.imresize(img, \ scale=config.VIDEO_FRAME_RESIZE_SCALING_FACTOR); im = img; pp = multi_scale_harris.multi_scale_harris(im, nos, disp=0); # n=0:nos-1 #harlocs = pp harlocs.append(pp); multi_scale_harris.StoreMultiScaleHarrisFeatures( \ harlocsFolder + "/" + fileNamePrefix + "%05d%s" % \ (counter, fileNameExtension), pp); counter += counterStep; # If we try to seek to a frame out-of-bounds frame it gets to the last one if config.OCV_OLD_PY_BINDINGS: capture.set(cv2.cv.CV_CAP_PROP_POS_FRAMES, counter); else: capture.set(cv2.CAP_PROP_POS_FRAMES, counter); #indexHarloc += 1; t2 = float(cv2.getTickCount()); myTime = (t2 - t1) / cv2.getTickFrequency(); common.DebugPrint("ComputeHarlocs(): computing the multiscale harlocs " \ "took %.6f [sec]" % myTime); #common.DebugPrint("ComputeHarlocs(): len(harlocs) = %s" % str(len(harlocs))); if False: for i, h in enumerate(harlocs): #common.DebugPrint("ComputeHarlocs(): len(harlocs[%d]) = %d" % \ # (i, len(h))); multi_scale_harris.StoreMultiScaleHarrisFeatures( harlocsFolder + "/" + fileNamePrefix + "%05d.txt" % i, h); #if False: if common.MY_DEBUG_STDOUT: common.DebugPrint("ComputeHarlocs(): harlocs = %s" % str(harlocs)); return harlocs;
def multi_scale_harris_Evangelidis(im, nos, disp): common.DebugPrint("Entered multi_scale_harris_Evangelidis(nos=%d, disp=%s)" % \ (nos, str(disp))) #tic #if size(im, 3) == 3: # im = rgb2gray(im) tMSH1 = float(cv2.getTickCount()) if im.ndim == 3: im = common.ConvertImgToGrayscale(im) #im = im2double(im) # From https://stackoverflow.com/questions/10873824/how-to-convert-2d-float-numpy-array-to-2d-int-numpy-array: #DO NOT USE: im = im.astype(np.uint8); # This messes up tremendously the computation of harlocs #im = im.astype(float); im = im.astype(np.float32) #!!!!COR is an UNused variable #COR = zeros(size(im,1), size(im,2), size(im,3)) #COR = np.zeros((im.shape[0], im.shape[1], im.shape[1])); # scale values sigma_0 = 1.2 #n=[0:nos-1]; %scale levels n = np.array(range(nos)) #scale levels #sigma_D=sqrt(1.8).^n*sigma_0 sigma_D = math.sqrt(1.8)**n * sigma_0 #points=[]; points = np.array([]) #scn=sigma_D.^4; scn = sigma_D**4 #for i=1:length(sigma_D) #for i in range(1, len(sigma_D) + 1): for i in range(1, nos + 1): common.DebugPrint("multi_scale_harris_Evangelidis(): i = %s" % (str(i))) #sd=sigma_D(i); %differentiation (local) scale sd = sigma_D[i - 1] #%differentiation (local) scale #si=sd/.5; %integration scale si = sd / 0.5 #integration scale w = 3 * sd # size for gaussian kernel to compute derivatives: 3 times local_scale r_w = int(round(w)) common.DebugPrint("multi_scale_harris_Evangelidis(): r_w = %s" % \ (str(r_w))) #if mod(round(w),2): if (r_w % 2) == 1: #[xco,yco] = meshgrid(-round(w):round(w),-round(w):round(w)) mRange = range(-r_w, r_w + 1) xco, yco = Matlab.meshgrid(mRange, mRange) else: #[xco,yco] = meshgrid(-(round(w)+1):round(w)+1,-(round(w)+1):round(w)+1) mRange = range(-(r_w + 1), r_w + 2) xco, yco = Matlab.meshgrid(mRange, mRange) if common.MY_DEBUG_STDOUT: common.DebugPrint("multi_scale_harris_Evangelidis(): xco = %s" % \ str(xco)) common.DebugPrint("multi_scale_harris_Evangelidis(): yco = %s" % \ str(yco)) # Note: even for HD frames, xco.shape = (11, 11) (yco the same) #arg = -(xco.*xco + yco.*yco) / (2*sd*sd) #arg = -(xco * xco + yco * yco) / (2.0 * sd * sd); arg = -(xco**2 + yco**2) / (2.0 * sd * sd) if common.MY_DEBUG_STDOUT: common.DebugPrint("multi_scale_harris_Evangelidis(): arg = %s" % \ str(arg)) #%2d gaussian kernel """ From http://www.mathworks.com/help/matlab/ref/exp.html: "exp(X) returns the exponential for each element of array X." """ #g=exp(arg)/(2*pi*sd^2); #2d gaussian kernel g = np.exp(arg) / (2.0 * math.pi * pow(sd, 2)) if common.MY_DEBUG_STDOUT: common.DebugPrint("multi_scale_harris_Evangelidis(): g = %s" % \ str(g)) # normalize to suppress any gain #if sum(g(:))~=0: g_sum = g.sum() #if abs(g.sum()) > 1.0e-6: if abs(g_sum) > 1.0e-6: #g = g / sum(g(:)); #g = g / float(g.sum()); g /= float(g_sum) if common.MY_DEBUG_STDOUT: common.DebugPrint("multi_scale_harris_Evangelidis(): sd = %s" % str(sd)) common.DebugPrint("multi_scale_harris_Evangelidis(): w = %s" % str(w)) """ #%Instead of computing derivatives in the filtered image, we filter the image with the derivatives of the kernel. """ #% kernels for gaussian derivatives #gx=-xco.*g/(sd*sd); gx = -xco * g / float(sd * sd) #gy=-yco.*g/(sd*sd); gy = -yco * g / float(sd * sd) if common.MY_DEBUG_STDOUT: common.DebugPrint("multi_scale_harris_Evangelidis(): gx = %s" % \ str(gx)) common.DebugPrint("multi_scale_harris_Evangelidis(): gy = %s" % \ str(gy)) """ multi_scale_harris_Evangelidis(): arg.shape = (11, 11) multi_scale_harris_Evangelidis(): g.shape = (11, 11) multi_scale_harris_Evangelidis(): gx.shape = (11, 11) multi_scale_harris_Evangelidis(): gy.shape = (11, 11) multi_scale_harris_Evangelidis(): gi.shape = (15, 15) """ common.DebugPrint("multi_scale_harris_Evangelidis(): xco.shape = %s" % str(xco.shape)) common.DebugPrint("multi_scale_harris_Evangelidis(): yco.shape = %s" % str(yco.shape)) common.DebugPrint("multi_scale_harris_Evangelidis(): arg.shape = %s" % str(arg.shape)) common.DebugPrint("multi_scale_harris_Evangelidis(): g.shape = %s" % str(g.shape)) common.DebugPrint("multi_scale_harris_Evangelidis(): gx.shape = %s" % str(gx.shape)) common.DebugPrint("multi_scale_harris_Evangelidis(): gy.shape = %s" % str(gy.shape)) #%compute the derivatives #Ix = imfilter(im, gx, 'replicate'); Ix = Matlab.imfilter(im, gx, "replicate") #Iy = imfilter(im, gy, 'replicate'); Iy = Matlab.imfilter(im, gy, "replicate") #% Alex: Ix and Iy have the same size as im #if True: if common.MY_DEBUG_STDOUT: common.DebugPrint("multi_scale_harris_Evangelidis(): Ix = %s" % \ str(Ix)) common.DebugPrint("multi_scale_harris_Evangelidis(): Iy = %s" % \ str(Iy)) #% gaussian kernel to compute 2nd moment matrix #if mod(floor(6*si),2) %size: six times the integration scale if int(math.floor( 6 * si)) % 2 == 1: #size: six times the integration scale #gi = fspecial('ga',max(1,fix(6*si)), si) gi = Matlab.fspecial("ga", max(1, Matlab.fix(6 * si)), si) else: #gi = fspecial('ga',max(1,fix(6*si)+1), si) gi = Matlab.fspecial("ga", max(1, Matlab.fix(6 * si) + 1), si) #if True: if common.MY_DEBUG_STDOUT: common.DebugPrint("multi_scale_harris_Evangelidis(): gi = %s" % \ str(gi)) common.DebugPrint("multi_scale_harris_Evangelidis(): gi.shape = %s" % \ str(gi.shape)) #Ix2 = imfilter(Ix.^2, gi, 'replicate'); Ix2 = Matlab.imfilter(Ix**2, gi, "replicate") #Iy2 = imfilter(Iy.^2, gi, 'replicate'); Iy2 = Matlab.imfilter(Iy**2, gi, "replicate") #Ixy = imfilter(Ix.*Iy, gi, 'replicate'); Ixy = Matlab.imfilter(Ix * Iy, gi, "replicate") #% Alex: Ix2, Iy2 and Ixy have the same size as im #if True: if common.MY_DEBUG_STDOUT: common.DebugPrint("multi_scale_harris_Evangelidis(): Ix2 = %s" % \ str(Ix2)) common.DebugPrint("multi_scale_harris_Evangelidis(): Iy2 = %s" % \ str(Iy2)) common.DebugPrint("multi_scale_harris_Evangelidis(): Ixy = %s" % \ str(Ixy)) common.DebugPrint("multi_scale_harris_Evangelidis(): Ix2.dtype = %s" % \ str(Ix2.dtype)) common.DebugPrint("multi_scale_harris_Evangelidis(): Iy2.dtype = %s" % \ str(Iy2.dtype)) common.DebugPrint("multi_scale_harris_Evangelidis(): Ixy.dtype = %s" % \ str(Ixy.dtype)) #%% Cornerness measure #% Noble measure. #% #% M = (Ix2.*Iy2 - Ixy.^2)./(Ix2 + Iy2 + eps); #% Harris measure. #M = (Ix2.*Iy2 - Ixy.^2) - .06*(Ix2 + Iy2).^2; M = (Ix2 * Iy2 - Ixy**2) - 0.06 * (Ix2 + Iy2)**2 if common.MY_DEBUG_STDOUT: common.DebugPrint("multi_scale_harris_Evangelidis(): M.dtype = %s" % \ str(M.dtype)) common.DebugPrint("multi_scale_harris_Evangelidis(): M = %s" % \ str(M)) #% Alex: scn is a vector - see definition above #% Alex: M has the same size as im #M = scn(i)*M; M = scn[i - 1] * M #thresh = 0.001*max(M(:)); thresh = 0.001 * M.max() #% imagesc(M==abs(M));axis on; #% colorbar #% pause """ %keep points that are the maximum in a neighborhood of radius=round(3*si/2) and are above thresh %non-maximum supression and subpixel refinement """ #[r,c, rsubp, csubp] = my_nms(M, round(3*si/2), thresh); r, c, rsubp, csubp = my_nms(M, round(3 * si / 2.0), thresh) if common.MY_DEBUG_STDOUT: common.DebugPrint( "multi_scale_harris_Evangelidis(): r.shape = %s" % str(r.shape)) common.DebugPrint("multi_scale_harris_Evangelidis(): r = %s" % str(r)) common.DebugPrint( "multi_scale_harris_Evangelidis(): c.shape = %s" % str(c.shape)) common.DebugPrint("multi_scale_harris_Evangelidis(): c = %s" % str(c)) common.DebugPrint( "multi_scale_harris_Evangelidis(): rsubp.shape = %s" % str(rsubp.shape)) common.DebugPrint("multi_scale_harris_Evangelidis(): rsubp = %s" % str(rsubp)) common.DebugPrint( "multi_scale_harris_Evangelidis(): csubp.shape = %s" % str(csubp.shape)) common.DebugPrint("multi_scale_harris_Evangelidis(): csubp = %s" % str(csubp)) #% Alex: r,c, rsubp, csubp seem to always be the same size - and the #% size I've seen is 56 * 1???? #pp=[rsubp, csubp, i*ones(size(r,1),1)]; pp = np.c_[rsubp, csubp, i * np.ones((r.shape[0], 1))] #% Alex: here we add more rows (pp) to points below the existing rows of #% points #points=[points; pp]; common.DebugPrint( "multi_scale_harris_Evangelidis(): points.shape = %s" % str(points.shape)) common.DebugPrint("multi_scale_harris_Evangelidis(): pp.shape = %s" % str(pp.shape)) if points.size == 0: # Avoiding exception: "ValueError: arrays must have same number of dimensions" points = pp else: points = np.r_[points, pp] #toc if disp: assert False # not implemented the display of Harris features """ figure; imshow(im,[]) #hold on title('corners detected') for i = range(1, size(points,1) + 1): rectangle('Position',[points(i,2)-3*points(i,3),points(i,1)-3*points(i,3),... 2*3*points(i,3),2*3*points(i,3)],'Curvature',[1,1],'EdgeColor','w','LineWidth',2) plot(points(i,2),points(i,1),'r+') """ if common.MY_DEBUG_STDOUT: common.DebugPrint("multi_scale_harris_Evangelidis(): points = %s" % str(points)) common.DebugPrint("multi_scale_harris_Evangelidis(): points.shape = %s" % str(points.shape)) tMSH2 = float(cv2.getTickCount()) myTime = (tMSH2 - tMSH1) / cv2.getTickFrequency() print( "multi_scale_harris_Evangelidis(): multi_scale_harris_Evangelidis() took %.6f [sec]" % myTime) return points
def multiscale_quad_retrieval(r_quadsTree, r_harlocs, q_harlocs, md_threshold, st_threshold, \ all_ori, all_id, all_max, all_cen, nos, scale_index, cropflag, \ sequence): common.DebugPrint("Entered multiscale_quad_retrieval(): " \ "md_threshold = %s, st_threshold = %s." % \ (str(md_threshold), \ str(st_threshold))); assert len(r_harlocs) != 0; assert len(q_harlocs) != 0; try: Votes_space = np.load("Votes_space%d.npz" % scale_index)['arr_0']; HH = np.load("HH%d.npz" % scale_index)['arr_0']; return Votes_space, HH; except: common.DebugPrintErrorTrace(); if common.MY_DEBUG_STDOUT and DBGPRINT: common.DebugPrint("multiscale_quad_retrieval(): r_quadsTree = %s" % \ str(r_quadsTree)); common.DebugPrint("multiscale_quad_retrieval(): len(r_harlocs) = %d" % len(r_harlocs)); common.DebugPrint("multiscale_quad_retrieval(): r_harlocs = %s" % str(r_harlocs)); common.DebugPrint("multiscale_quad_retrieval(): q_harlocs = %s" % str(q_harlocs)); common.DebugPrint("multiscale_quad_retrieval(): md_threshold = %s" % str(md_threshold)); print("multiscale_quad_retrieval(): st_threshold = %s" % str(st_threshold)); #common.DebugPrint("multiscale_quad_retrieval(): all_ori, all_id, all_max, all_cen, nos, scale_index, cropflag = %s" % str(all_ori, all_id, all_max, all_cen, nos, scale_index, cropflag)); common.DebugPrint("multiscale_quad_retrieval(): all_id = %s" % str(all_id)); common.DebugPrint("multiscale_quad_retrieval(): all_id.shape = %s" % (str(all_id.shape))); #common.DebugPrint("multiscale_quad_retrieval(): all_max, all_cen, nos, scale_index, cropflag = %s" % str(all_max, all_cen, nos, scale_index, cropflag)); #common.DebugPrint("multiscale_quad_retrieval(): all_max = %s" % str(all_max)); #common.DebugPrint("multiscale_quad_retrieval(): all_cen, nos, scale_index, cropflag = %s" % str(all_cen, nos, scale_index, cropflag)); common.DebugPrint("multiscale_quad_retrieval(): sequence = %s" % str(sequence)); print("multiscale_quad_retrieval(): cropflag = %s" % str(cropflag)); t1 = float(cv2.getTickCount()); if scale_index > nos: assert scale_index <= nos; #error('Wrong scale index or number-of-scales'); #QD = dir([q_path "multiharlocs*.mat"]) #QD = [q_path + "multiharlocs*.mat"] #QD = q_harlocs; #RD = dir([r_path "multiharlocs*.mat"]) #RD = [r_path + "multiharlocs*.mat"] #RD = r_harlocs; #TODO: take out RD_start #RD_start = str2num(RD(1).name(end - 9 : end - 4)) #RD_start = int(RD[0][-9 : -4]) RD_start = 0; #RD_end = str2num(RD(end).name(end - 9 : end - 4)) #RD_end = int(RD[-1][-9 : -4]) #RD_end = len(RD) - 1; RD_end = len(r_harlocs) - 1; if False: # n_d not used anywhere #n_d = hist(all_id, RD_start : RD_end) #n_d = hist[all_id, RD_start : RD_end] n_d = Matlab.hist(x=all_id, \ binCenters=np.array(range(RD_start, RD_end + 1)) ); #cross_indices = np.zeros( (len(QD), 2) ); cross_indices = np.zeros( (len(q_harlocs), 2) ); j = 1; #tic #ORI = np.array([]); # ORI NOT used anywhere """ Inspired from https://stackoverflow.com/questions/17559140/matlab-twice-as-fast-as-numpy BUT doesn't help in this case: Votes_space = np.asfortranarray(np.zeros( (len(RD), len(QD)) )); """ #Votes_space = np.zeros( (len(RD), len(QD)) ); Votes_space = np.zeros( (len(r_harlocs), len(q_harlocs)) ); # Make a distinct copy of HH from Votes_space... #HH = Votes_space.copy().astype(np.int16); #Votes_space + 0; #HH = np.zeros((len(RD), len(QD)), dtype=np.int8); HH = np.zeros((len(r_harlocs), len(q_harlocs)), dtype=np.int8); #!!!!TODO use MAYBE even np.bool - OR take it out #common.DebugPrint("multiscale_quad_retrieval(): Votes_space = %s,\n HH = %s" % (str(Votes_space), str(HH))) tolers = 0.1 - float(scale_index) / 100.0; # it helps to make more strict the threshold as the scale goes up # tolers = 0.15 - float(scale_index) / 100.0; MAXDIS = 3 + scale_index; MAXORI = 0.25; """ !!!!TODO TODO: I am using multiprocessing.Poll and return votes; the dispatcher assembles the results, but the results are NOT the same with the serial case - although they look pretty decent, but they seem to be suboptimal - dp_Alex returns suboptimal cost path for USE_MULTITHREADING == True instead of False. (Note: running under the same preconditions multiscale_quad_retrieval I got the same results in dp_Alex(). """ if False: #config.USE_MULTITHREADING == True: global g; g.r_quadsTree = r_quadsTree; g.r_harlocs = r_harlocs; g.q_harlocs = q_harlocs; g.md_threshold = md_threshold; g.st_threshold = st_threshold; g.all_ori = all_ori; g.all_id = all_id; g.all_max = all_max; g.all_cen = all_cen; g.nos = nos; g.scale_index = scale_index; g.cropflag = cropflag; g.sequence = sequence; g.RD_start = RD_start; g.RD_end = RD_end; g.MAXDIS = MAXDIS; g.MAXORI = MAXORI; g.tolers = tolers; """ Start worker processes to use on multi-core processor (able to run in parallel - no GIL issue if each core has it's own VM) """ pool = multiprocessing.Pool(processes=config.numProcesses); print("multiscale_quad_retrieval(): Spawned a pool of %d workers" % \ config.numProcesses); listParams = range(0, len(q_harlocs)); #!!!!TODO: use counterStep, config.initFrame[indexVideo] #res = pool.map(IterationStandaloneMQR, listParams); # See https://docs.python.org/2/library/multiprocessing.html#module-multiprocessing.pool res = pool.map(func=IterationStandaloneMQR, iterable=listParams, \ chunksize=1); print("Pool.map returns %s" % str(res)); #x0.size + 1 """ From https://medium.com/building-things-on-the-internet/40e9b2b36148 close the pool and wait for the work to finish """ pool.close(); pool.join(); # Doing the "reduce" phase after the workers have finished :) assert len(res) == len(q_harlocs); for queryFrame, resE in enumerate(res): resEIndex = resE[0]; resE = resE[1]; assert resEIndex == queryFrame; # Gives: "ValueError: output operand requires a reduction, but reduction is not enabled" #Votes_space[:, queryFrame - 1] = votes; Votes_space[:, queryFrame] = resE; for queryFrame in range(len(q_harlocs)): if cropflag == 0: HH[:, queryFrame] = 1; else: """ HH[:, queryFrame] = spatial_consistency.spatial_consistency(space_xy, \ qcen, len(RD), st_threshold, cropflag); """ HH[:, queryFrame] = spatial_consistency.spatial_consistency(space_xy, \ qcen, len(r_harlocs), st_threshold, cropflag); try: np.savez_compressed("Votes_space%d" % scale_index, Votes_space); np.savez_compressed("HH%d" % scale_index, HH); except: common.DebugPrintErrorTrace(); return Votes_space, HH; """ We substitute q - 1 with q, since we want to number arrays from 0 (not from 1 like in Matlab). """ #for q=1:length(QD) #for q in range(1, len(QD) + 1): #for queryFrame in range(len(QD)): for queryFrame in range(len(q_harlocs)): common.DebugPrint("multiscale_quad_retrieval(): Starting iteration queryFrame = %d" % queryFrame); # tic """ str1=['load ' q_path QD(q).name] eval(str1) """ """ We make pp reference the desired multiharloc list for the query video frame queryFrame """ pp = q_harlocs[queryFrame]; #pp = np.array(pp); #common.DebugPrint("multiscale_quad_retrieval(): pp = %s" % str(pp)); #[qout,qcen,qmaxdis,qori]=findquads(pp(pp(:,3)==scale_index,1:2),md_threshold,0); points = pp[pp[:, 2] == scale_index, 0:2]; qout, qcen, qmaxdis, qori = findquads.findquads(points, md_threshold, 0); if common.MY_DEBUG_STDOUT and DBGPRINT: print("multiscale_quad_retrieval(): queryFrame = %d, " \ "qout.shape (number of quads for query frame queryFrame) = %s" % \ (queryFrame, str(qout.shape))); # disp([num2str(q) ' of ' num2str(length(QD)) ' -> ' num2str(size(qout,1)) ' quads']) #space_xy=zeros(size(qcen,1),2*length(RD))+nan; #space_xy = np.zeros( (qcen.shape[0], 2 * len(RD)) ) + np.nan; space_xy = np.zeros( (qcen.shape[0], 2 * len(r_harlocs)) ) + np.nan; # votes=zeros(length(RD),1) #votes=zeros(length(RD),length(tolers)); #votes = np.zeros( (len(RD), 1) ); votes = np.zeros( (len(r_harlocs), 1) ); #nep = np.array([]); #m_points = np.array([]); assert isinstance(tolers, float); if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): quads of query frame %d are: " % queryFrame); common.DebugPrint(" qout = %s" % str(qout)); """ Alex: for each quad (4 floats) of the query frame from Harris feature of scale scale_index Note: all_id stores the reference frame id for each quad descriptor. """ """ We substitute queryFrameQuad - 1 with queryFrameQuad, since we want to number arrays from 0 (not from 1 like in Matlab). """ #for queryFrameQuad in range(1, qout.shape[0] + 1): for queryFrameQuad in range(qout.shape[0]): common.DebugPrint("multiscale_quad_retrieval(): Starting iteration queryFrameQuad = %d" % queryFrameQuad); """ Matlab's polymorphism is really bugging here: although it's normally a float, tolers is considered to be a size 1 vector... so len(tolers) == 1 """ #for tol_i in range(1, len(tolers) + 1): # tol = tolers[tol_i - 1] """ We substitute tol_i - 1 with tol, since we want to number arrays from 0 (not from 1 like in Matlab). """ #for tol_i in range(1, 1 + 1): for tol_i in range(1): tol = tolers; """ # TODO: done below - take out this dbg print if DBGPRINT: common.DebugPrint("multiscale_quad_retrieval(): " \ "qout[queryFrameQuad, :] = %s" % \ str(qout[queryFrameQuad, :])) """ #% default for first PAMI with tol= 0.1 approximately # NOTE: SciPy's KDTree finds a few more results, in some cases, # than the Matlab code from Evangelidis. #idx, di = kdtree_ball_query(tree, qout(i, :), tol) #idx, distKD = kdtree_ball_query(tree, qout[i - 1, :], tol) #idx, di = tree.query(x=xQuery, k=4) #resPoints = [data[i] for i in resBallIndices] # tol is a scalar representing the radius of the ball if config.KDTREE_IMPLEMENTATION == 0: idx = r_quadsTree.query_ball_point(qout[queryFrameQuad, :], tol); elif config.KDTREE_IMPLEMENTATION == 1: #pt = qout[queryFrameQuad - 1, :].astype(np.float32); pt = qout[queryFrameQuad, :]; pt = np.array([[pt[0], pt[1], pt[2], pt[3]]], dtype=np.float32); retval, idx, dists = r_quadsTree.radiusSearch( \ query=pt, \ radius=(tol**2), \ maxResults=NUM_MAX_ELEMS, \ params=search_params); if common.MY_DEBUG_STDOUT and DBGPRINT: """ common.DebugPrint("multiscale_quad_retrieval(): " \ "retval (number NNs) = %s" % str(retval)); """ common.DebugPrint( \ "multiscale_quad_retrieval(): radiusSearch's retval " \ "(at queryFrame=%d, queryFrameQuad=%d) is %d" % (queryFrame, queryFrameQuad, retval)); idx = idx[0]; dists = dists[0]; """ Note: retval is the number of neighbors returned from the radiusSearch(). But the idx and the dists can have more elements than the returned retval. """ idx = idx[: retval]; dists = dists[: retval]; if common.MY_DEBUG_STDOUT and DBGPRINT: print("multiscale_quad_retrieval(): " \ "qout[queryFrameQuad, :] = %s" % str(qout[queryFrameQuad, :])); print("multiscale_quad_retrieval(): " \ "idx = %s" % str(idx)); print("multiscale_quad_retrieval(): " \ "dists = %s" % str(dists)); print("multiscale_quad_retrieval(): " \ "tol = %s" % str(tol)); if config.KDTREE_IMPLEMENTATION == 0: print("multiscale_quad_retrieval(): " \ "r_quadsTree.data[idx] = %s" % \ str(r_quadsTree.data[idx])); # We print the distances to the points returned in idx if common.MY_DEBUG_STDOUT and DBGPRINT: # This is just for debugging purposes a = qout[queryFrameQuad, :]; if config.KDTREE_IMPLEMENTATION == 0: for myI, index in enumerate(idx): b = r_quadsTree.data[index]; """ if False: common.DebugPrint("multiscale_quad_retrieval(): distance to " \ "%d point (%s) inside ball = %.4f" % \ (myI, str(b), npla.norm(a - b))); """ else: pass; idx = np.array(idx); #if False: if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " \ "all_max.shape = %s" % str(all_max.shape)); common.DebugPrint("multiscale_quad_retrieval(): " \ "qmaxdis.shape = %s" % str(qmaxdis.shape)); common.DebugPrint("multiscale_quad_retrieval(): " \ "qmaxdis = %s" % str(qmaxdis)); common.DebugPrint("multiscale_quad_retrieval(): " \ "qori.shape = %s" % str(qori.shape)); common.DebugPrint("multiscale_quad_retrieval(): " \ "qori = %s" % str(qori)); #dis_idx=abs(qmaxdis(i)-all_max(idx))<MAXDIS; if len(idx) == 0: # NOT A GOOD IDEA: continue; #idx = np.array([]); dis_idx = np.array([]); ori_idx = np.array([]); else: if common.MY_DEBUG_STDOUT and DBGPRINT: print("multiscale_quad_retrieval(): " \ "queryFrameQuad = %s" % str(queryFrameQuad)); print("multiscale_quad_retrieval(): " \ "all_max[idx] = %s" % str(all_max[idx])); print("multiscale_quad_retrieval(): " \ "qmaxdis[queryFrameQuad] = %s" % str(qmaxdis[queryFrameQuad])); if USE_GPS_COORDINATES: # We look only at a part of the reference video """ Since in some cases the video temporal alignment is difficult to do due to similar portions in the trajectory (see the drone videos, clip 3_some_lake) we "guide" the temporal alignment by restricting the reference frame search space - this is useful when we have the geolocation (GPS) coordinate for each frame. """ if common.MY_DEBUG_STDOUT and DBGPRINT: print("multiscale_quad_retrieval(): " \ "all_id = %s" % str(all_id)); if True: #assert (all_id.ndim == 2) and (all_id.shape[1] == 1); if all_id.ndim == 2: #!!!!TODO TODO: put this at the beginning of the function assert all_id.shape[1] == 1; """ We flatten the array all_id Note: We don't use order="F" since it's basically 1-D array """ all_id = np.ravel(all_id); #!!!!TODO: put start and end frame in config - or compute it from geolocation sub_idx = np.logical_and( (all_id[idx] >= 2030 - 928), \ (all_id[idx] <= 2400 - 928) ); idx = idx[sub_idx]; if common.MY_DEBUG_STDOUT and DBGPRINT: print("multiscale_quad_retrieval(): " \ "all_id = %s" % str(all_id)); print("multiscale_quad_retrieval(): " \ "sub_idx = %s" % str(sub_idx)); print("multiscale_quad_retrieval(): " \ "idx = %s" % str(idx)); if FILTER: dis_idx = np.abs(qmaxdis[queryFrameQuad] - all_max[idx]) < MAXDIS; #if False: if common.MY_DEBUG_STDOUT: """ common.DebugPrint("multiscale_quad_retrieval(): " \ "idx = %s" % str(idx)); """ common.DebugPrint("multiscale_quad_retrieval(): " \ "dis_idx = %s" % str(dis_idx)); #idx=idx(dis_idx) idx = idx[dis_idx]; #if False: if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " \ "idx (after idx = idx[dis_idx]) = %s" % str(idx)); if FILTER: #ori_idx=abs(qori(i)-all_ori(idx))<MAXORI; ori_idx = np.abs(qori[queryFrameQuad] - all_ori[idx]) < MAXORI; #if False: if common.MY_DEBUG_STDOUT: """ common.DebugPrint("multiscale_quad_retrieval(): " \ "all_ori = %s" % str(all_ori)); common.DebugPrint("multiscale_quad_retrieval(): " \ "qori[queryFrameQuad] = %s" % str(qori[queryFrameQuad])); """ common.DebugPrint("multiscale_quad_retrieval(): " \ "ori_idx = %s" % str(ori_idx)); #idx=idx(ori_idx); idx = idx[ori_idx]; # IMPORTANT ################################################### # IMPORTANT ################################################### # IMPORTANT ################################################### #% spatio-temporal consistency # IMPORTANT ################################################### # IMPORTANT ################################################### # IMPORTANT ################################################### #if numel(idx) > 0: if idx.size > 0: if cropflag == 0: if FILTER: """ Alex: this is a simple procedure of eliminating False Positive (FP) matches, as presented in Section 4.2 of TPAMI 2013 paper. Basically it filters out quad matches that have centroids st_threshold away from the query quad. Note: all_cen are the controids of all reference quads. """ dy = qcen[queryFrameQuad, 0] - all_cen[idx, 0]; dx = qcen[queryFrameQuad, 1] - all_cen[idx, 1]; #D=dy.^2+dx.^2; D = dy**2 + dx**2; co_idx = D < pow(st_threshold, 2); idx = idx[co_idx]; else: """ We substitute iii - 1 with iii, since we want to number arrays from 0 (not from 1 like in Matlab). """ #for iii in range(1, len(idx) + 1): for iii in range(len(idx)): #space_xy(i,(all_id(idx(iii))-RD_start)*2+1:(all_id(idx(iii))-RD_start)*2+2) = all_cen(idx(iii),:) space_xy[queryFrameQuad, \ (all_id[idx[iii]] - RD_start) * 2: (all_id[idx[iii] - 1] - RD_start) * 2 + 1] = \ all_cen[idx[iii], :]; #hh=hist(all_id(idx),RD_start:RD_end); # It has to be an np.array because we multiply it with a scalar histoRange = np.array(range(RD_start, RD_end + 1)); hh = Matlab.hist(x=all_id[idx], binCenters=histoRange); #if False: #if True: if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " \ "hh = %s" % (str(hh))); common.DebugPrint("multiscale_quad_retrieval(): " \ "hh.shape = %s" % (str(hh.shape))); """ common.DebugPrint("multiscale_quad_retrieval(): " \ "all_id = %s" % (str(all_id))); """ common.DebugPrint("multiscale_quad_retrieval(): " \ "all_id.shape = %s" % (str(all_id.shape))); common.DebugPrint("multiscale_quad_retrieval(): " \ "idx = %s" % (str(idx))); common.DebugPrint("multiscale_quad_retrieval(): " \ "idx.shape = %s" % (str(idx.shape))); # % nz can be computed more optimally #nz=find(hh~=0); # nz can be computed more optimally # np.nonzero() always returns a tuple, even if it contains 1 element since hh has only 1 dimension nz = np.nonzero(hh != 0)[0]; #if False: if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " \ "nz = %s" % (str(nz))); common.DebugPrint("multiscale_quad_retrieval(): " \ "nz.shape = %s" % (str(nz.shape))); #if numel(nz) > 0 if nz.size > 0: #%%----text-retrieval-like #votes(nz, tol_i) = votes(nz, tol_i) + log10(length(RD) / (length(nz)))^2 #PREVIOUSLY #myVal = pow(math.log10(float(len(RD)) / len(nz)), 2); myVal = pow(math.log10(float(len(r_harlocs)) / len(nz)), 2); """ try: myVal = pow(math.log10(float(len(r_harlocs)) / len(nz)), 2); except: print("Error: len=%d len(nz)=%d nz.size=%d" % \ (len(r_harlocs), len(nz), nz.size)); common.DebugPrintErrorTrace(); """ #if False: if common.MY_DEBUG_STDOUT: """ common.DebugPrint("multiscale_quad_retrieval(): " \ "len(RD) = %d" % len(RD)); """ common.DebugPrint("multiscale_quad_retrieval(): " \ "len(r_harlocs) = %d" % len(r_harlocs)); common.DebugPrint("multiscale_quad_retrieval(): " \ "len(nz) = %d" % len(nz)); common.DebugPrint("multiscale_quad_retrieval(): " \ "myVal = %.5f" % myVal); # PREVIOUSLY votes[nz, tol_i] = votes[nz, tol_i] + myVal; # votes(nz)=votes(nz)+log10(length(RD)/(length(nz))); # votes(nz)=votes(nz)+1; if common.MY_DEBUG_STDOUT and DBGPRINT: """ common.DebugPrint("multiscale_quad_retrieval(): " \ "Votes_space.shape = %s" % (str(Votes_space.shape))); common.DebugPrint("multiscale_quad_retrieval(): " \ "votes.shape = %s" % (str(votes.shape))); """ print("multiscale_quad_retrieval(): " \ "votes.shape = %s" % (str(votes.shape))); if (np.abs(votes) < 1.0e-10).all(): print( \ "multiscale_quad_retrieval(): votes = 0 (all zeros)"); else: print("multiscale_quad_retrieval(): " \ "votes = %s" % (str(votes))); #Votes_space(:,q)=votes; # Gives: "ValueError: output operand requires a reduction, but reduction is not enabled" #Votes_space[:, queryFrame - 1] = votes; # Note: since votes is basically a 1-D vector, we don't use the Fortran order Votes_space[:, queryFrame] = np.ravel(votes); # order="F"); if cropflag == 0: HH[:, queryFrame] = 1; else: """ HH[:, queryFrame] = spatial_consistency.spatial_consistency(space_xy, \ qcen, len(RD), st_threshold, cropflag); """ HH[:, queryFrame] = spatial_consistency.spatial_consistency(space_xy, \ qcen, len(r_harlocs), st_threshold, cropflag); if common.MY_DEBUG_STDOUT and DBGPRINT: print("multiscale_quad_retrieval(scale_index=%d): " \ "Votes_space =\n%s" % (scale_index, str(Votes_space))); try: np.savez_compressed("Votes_space%d" % scale_index, Votes_space); np.savez_compressed("HH%d" % scale_index, HH); except: common.DebugPrintErrorTrace(); t2 = float(cv2.getTickCount()); myTime = (t2 - t1) / cv2.getTickFrequency(); print("multiscale_quad_retrieval() took %.6f [sec]" % myTime); """ common.DebugPrint("multiscale_quad_retrieval(): " \ "%d corresponding frames retrieved in %.6f secs" % \ (len(q_harlocs), myTime)); """ return Votes_space, HH;
def ComputeHarlocs(capture, counterStep, folderName, fileNamePrefix, fileNameExtension=".csv", indexVideo=-1): print( \ "Entered ComputeHarlocs(capture=%s, counterStep=%d, folderName=%s, " \ "indexVideo=%d)" % \ (str(capture), counterStep, folderName, indexVideo)) harlocsFolder = config.VIDEOS_FOLDER + "/" + folderName t1 = float(cv2.getTickCount()) harlocs = [] if config.OCV_OLD_PY_BINDINGS: numFrames = int(capture.get(cv2.cv.CV_CAP_PROP_FRAME_COUNT)) else: numFrames = int(capture.get(cv2.CAP_PROP_FRAME_COUNT)) common.DebugPrint("ComputeHarlocs(): numFrames = %d" % numFrames) if not os.path.exists(harlocsFolder): os.makedirs(harlocsFolder) else: #!!!!TODO: check that the loaded Harlocs are complete - same frame numbers as in the videos # Folder with precomputed Harris features exists folderContent = os.listdir(harlocsFolder) sortedFolderContent = sorted(folderContent) for fileName in sortedFolderContent: pathFileName = harlocsFolder + "/" + fileName """ common.DebugPrint("ComputeHarlocs(): pathFileName = %s" % pathFileName); common.DebugPrint("ComputeHarlocs(): fileName = %s" % fileName); """ if os.path.isfile(pathFileName) and \ fileName.startswith(fileNamePrefix) and \ pathFileName.endswith(fileNameExtension): common.DebugPrint("ComputeHarlocs(): Loading %s" % pathFileName) harrisFeatures = multi_scale_harris.LoadMultiScaleHarrisFeatures( pathFileName) harlocs.append(harrisFeatures) if config.endFrame[indexVideo] == -1: assert (len(harlocs) + config.initFrame[indexVideo]) == numFrames #!!!!TODO: if condition is NOT met, give a nicer error, or redo computations of Harlocs else: assert (len(harlocs) + config.initFrame[indexVideo] ) == config.endFrame[indexVideo] + 1 #!!!!TODO: if condition is NOT met, give a nicer error, or redo computations of Harlocs return harlocs if config.USE_MULTITHREADING == True: global g g.captureQ = None # We need to reopen the capture device in each process, separately g.captureR = None # We need to reopen the capture device in each process, separately #g.capture = capture; g.harlocsFolder = harlocsFolder g.fileNamePrefix = fileNamePrefix g.fileNameExtension = fileNameExtension g.indexVideo = indexVideo if config.OCV_OLD_PY_BINDINGS: frameCount = int(capture.get(cv2.cv.CV_CAP_PROP_FRAME_COUNT)) else: frameCount = int(capture.get(cv2.CAP_PROP_FRAME_COUNT)) #listParams = range(frameCount); listParams = range(config.initFrame[indexVideo], frameCount, counterStep) common.DebugPrint("ComputeHarlocs(): frameCount = %d" % frameCount) print("ComputeHarlocs(): Spawning a pool of %d workers" % \ config.numProcesses) """ # DEBUG purposes ONLY - since when we use Pool() and call function, if # we have an error in the function the exception reported is very # vague... for i in listParams: IterationStandaloneMSH(i); #import time #time.sleep(1000); """ """ Start worker processes to use on multi-core processor (circumvent also the GIL issue). """ pool = multiprocessing.Pool(processes=config.numProcesses) print("ComputeHarlocs(): Spawned a pool of %d workers" % \ config.numProcesses) #res = pool.map(IterationStandaloneMSH, listParams); # See https://docs.python.org/2/library/multiprocessing.html#module-multiprocessing.pool res = pool.map(func=IterationStandaloneMSH, iterable=listParams, \ chunksize=1) print("Pool.map returns %s" % str(res)) """ From https://medium.com/building-things-on-the-internet/40e9b2b36148 close the pool and wait for the work to finish """ pool.close() pool.join() #!!!!TODO: do more efficient - don't load the results from the CSV files return ComputeHarlocs(capture, counterStep, folderName, \ fileNamePrefix, fileNameExtension, indexVideo) #return []; #indexHarloc = 0; while True: if config.OCV_OLD_PY_BINDINGS: framePos = capture.get(cv2.cv.CV_CAP_PROP_POS_FRAMES) else: """ From http://docs.opencv.org/modules/highgui/doc/reading_and_writing_images_and_video.html#videocapture-get: <<CV_CAP_PROP_POS_FRAMES 0-based index of the frame to be decoded/captured next.>> """ framePos = capture.get(cv2.CAP_PROP_POS_FRAMES) common.DebugPrint("ComputeHarlocs(): framePos = %d" % framePos) counter = int(framePos) #0 common.DebugPrint("ComputeHarlocs(): counter = %d" % counter) ret, img = capture.read() if common.MY_DEBUG_STDOUT: common.DebugPrint("ComputeHarlocs(): img = %s" % str(img)) if False and config.SAVE_FRAMES: fileName = config.IMAGES_FOLDER + "/img_%05d.png" % counter if not os.path.exists(fileName): #print "dir(img) = %s"% str(dir(img)) """ imgCV = cv.fromarray(img) cv2.imwrite(fileName, imgCV) """ cv2.imwrite(fileName, img) #if ret == False: #MatchFrames.counterQ == 3: if (ret == False) or ((counter > numFrames) or \ (config.endFrame[indexVideo] != -1 and \ counter > config.endFrame[indexVideo])): break if config.VIDEO_FRAME_RESIZE_SCALING_FACTOR != 1: img = Matlab.imresize(img, \ scale=config.VIDEO_FRAME_RESIZE_SCALING_FACTOR) im = img pp = multi_scale_harris.multi_scale_harris(im, nos, disp=0) # n=0:nos-1 #harlocs = pp harlocs.append(pp) multi_scale_harris.StoreMultiScaleHarrisFeatures( \ harlocsFolder + "/" + fileNamePrefix + "%05d%s" % \ (counter, fileNameExtension), pp) counter += counterStep # If we try to seek to a frame out-of-bounds frame it gets to the last one if config.OCV_OLD_PY_BINDINGS: capture.set(cv2.cv.CV_CAP_PROP_POS_FRAMES, counter) else: capture.set(cv2.CAP_PROP_POS_FRAMES, counter) #indexHarloc += 1; t2 = float(cv2.getTickCount()) myTime = (t2 - t1) / cv2.getTickFrequency() common.DebugPrint("ComputeHarlocs(): computing the multiscale harlocs " \ "took %.6f [sec]" % myTime) #common.DebugPrint("ComputeHarlocs(): len(harlocs) = %s" % str(len(harlocs))); if False: for i, h in enumerate(harlocs): #common.DebugPrint("ComputeHarlocs(): len(harlocs[%d]) = %d" % \ # (i, len(h))); multi_scale_harris.StoreMultiScaleHarrisFeatures( harlocsFolder + "/" + fileNamePrefix + "%05d.txt" % i, h) #if False: if common.MY_DEBUG_STDOUT: common.DebugPrint("ComputeHarlocs(): harlocs = %s" % str(harlocs)) return harlocs
def IterationStandaloneMQR(queryFrame): r_quadsTree = g.r_quadsTree; r_harlocs = g.r_harlocs; q_harlocs = g.q_harlocs; md_threshold = g.md_threshold; st_threshold = g.st_threshold; all_ori = g.all_ori; all_id = g.all_id; all_max = g.all_max; all_cen = g.all_cen; nos = g.nos; scale_index = g.scale_index; cropflag = g.cropflag; sequence = g.sequence; RD_start = g.RD_start; RD_end = g.RD_end; MAXDIS = g.MAXDIS; MAXORI = g.MAXORI; tolers = g.tolers; """ common.DebugPrint( \ "Entered IterationStandaloneMQR(): crossref=%s, captureQ=%s, "\ "captureR=%s, refined_crossref=%s, warp_p=%s, " "x0=%s, y0=%s, start=%s, t=%d, iWhile=%d." % \ (str(crossref), str(captureQ), str(captureR), \ str(g.refined_crossref), str(g.warp_p), \ str(g.x0), str(g.y0), str(g.start), g.t, iWhile)); common.DebugPrint("IterationStandalone(): id(g)=%s" % str(id(g))); """ # tic """ str1=['load ' q_path QD(q).name] eval(str1) """ """ We make pp reference the desired multiharloc list for the query video frame queryFrame """ pp = q_harlocs[queryFrame]; #pp = np.array(pp); #common.DebugPrint("multiscale_quad_retrieval(): pp = %s" % str(pp)); """ Alex: for the query frame queryFrame we retrieve, for scale scale_index, the harris features in var points. Then we build the quads from points. Then for each quad (4 float values) we query the corresponding scale kd-tree, and we get the indices. Then we build the histogram and compute idf, ....!!!! Note: scale is 1 for original frame resolution and the higher we go we have lower image resolutions (we go higher in the Guassian pyramid I think). """ #[qout,qcen,qmaxdis,qori]=findquads(pp(pp(:,3)==scale_index,1:2),md_threshold,0); points = pp[pp[:, 2] == scale_index, 0:2]; qout, qcen, qmaxdis, qori = findquads.findquads(points, md_threshold, 0); common.DebugPrint("multiscale_quad_retrieval(): queryFrame = %d, " \ "qout.shape = %s" % (queryFrame, str(qout.shape))); # disp([num2str(q) ' of ' num2str(length(QD)) ' -> ' num2str(size(qout,1)) ' quads']) #space_xy=zeros(size(qcen,1),2*length(RD))+nan; #space_xy = np.zeros( (qcen.shape[0], 2 * len(RD)) ) + np.nan; space_xy = np.zeros( (qcen.shape[0], 2 * len(r_harlocs)) ) + np.nan; # votes=zeros(length(RD),1) #votes=zeros(length(RD),length(tolers)); #votes = np.zeros( (len(RD), 1) ); votes = np.zeros( (len(r_harlocs), 1) ); #nep = np.array([]); #m_points = np.array([]); assert isinstance(tolers, float); """ We substitute queryFrameQuad - 1 with queryFrameQuad, since we want to number arrays from 0 (not from 1 like in Matlab). """ #for queryFrameQuad in range(1, qout.shape[0] + 1): for queryFrameQuad in range(qout.shape[0]): """ Matlab's polymorphism is really bugging here: although it's normally a float, tolers is considered to be a size 1 vector... so len(tolers) == 1 """ #for tol_i in range(1, len(tolers) + 1): # tol = tolers[tol_i - 1] """ We substitute tol_i - 1 with tol, since we want to number arrays from 0 (not from 1 like in Matlab). """ #for tol_i in range(1, 1 + 1): for tol_i in range(1): tol = tolers; #common.DebugPrint("multiscale_quad_retrieval(): qout[i - 1, :] = %s" % str(qout[i - 1, :])) #% default for first PAMI with tol= 0.1 approximately # NOTE: SciPy's KDTree finds a few more results, in some cases, # than the Matlab code from Evangelidis. #idx, di = kdtree_ball_query(tree, qout(i, :), tol) #idx, distKD = kdtree_ball_query(tree, qout[i - 1, :], tol) #idx, di = tree.query(x=xQuery, k=4) #resPoints = [data[i] for i in resBallIndices] # tol is a scalar representing the radius of the ball if config.KDTREE_IMPLEMENTATION == 0: idx = r_quadsTree.query_ball_point(qout[queryFrameQuad, :], tol); elif config.KDTREE_IMPLEMENTATION == 1: #pt = qout[queryFrameQuad - 1, :].astype(np.float32); pt = qout[queryFrameQuad, :]; pt = np.array([[pt[0], pt[1], pt[2], pt[3]]], dtype=np.float32); retval, idx, dists = r_quadsTree.radiusSearch( \ query=pt, \ radius=(tol**2), \ maxResults=NUM_MAX_ELEMS, \ params=search_params); if common.MY_DEBUG_STDOUT and DBGPRINT: """ common.DebugPrint("multiscale_quad_retrieval(): " \ "retval (number NNs) = %s" % str(retval)); """ common.DebugPrint( \ "multiscale_quad_retrieval(): radiusSearch's retval " \ "(at queryFrame=%d, queryFrameQuad=%d) is %d\n" % (queryFrame, queryFrameQuad, retval)); idx = idx[0]; dists = dists[0]; idx = idx[: retval]; dists = dists[: retval]; if common.MY_DEBUG_STDOUT and DBGPRINT: print("multiscale_quad_retrieval(): " \ "qout[queryFrameQuad, :] = %s" % str(qout[queryFrameQuad, :])); print("multiscale_quad_retrieval(): " \ "idx = %s" % str(idx)); print("multiscale_quad_retrieval(): " \ "tol = %s" % str(tol)); if config.KDTREE_IMPLEMENTATION == 0: print("multiscale_quad_retrieval(): " \ "r_quadsTree.data[idx] = %s" % \ str(r_quadsTree.data[idx])); # We print the distances to the points returned in idx a = qout[queryFrameQuad, :]; if False: #!!!! This is just for debugging purposes for myI, index in enumerate(idx): b = r_quadsTree.data[index]; """ if False: common.DebugPrint("multiscale_quad_retrieval(): distance to " \ "%d point (%s) inside ball = %.4f" % \ (myI, str(b), npla.norm(a - b))); """ idx = np.array(idx); #if False: if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " \ "all_max.shape = %s" % str(all_max.shape)); common.DebugPrint("multiscale_quad_retrieval(): " \ "qmaxdis.shape = %s" % str(qmaxdis.shape)); common.DebugPrint("multiscale_quad_retrieval(): " \ "qmaxdis = %s" % str(qmaxdis)); common.DebugPrint("multiscale_quad_retrieval(): " \ "qori.shape = %s" % str(qori.shape)); common.DebugPrint("multiscale_quad_retrieval(): " \ "qori = %s" % str(qori)); #dis_idx=abs(qmaxdis(i)-all_max(idx))<MAXDIS; if len(idx) == 0: # NOT A GOOD IDEA: continue; #idx = np.array([]); dis_idx = np.array([]); ori_idx = np.array([]); else: #if False: if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " \ "queryFrameQuad = %s" % str(queryFrameQuad)); common.DebugPrint("multiscale_quad_retrieval(): " \ "all_max[idx] = %s" % str(all_max[idx])); common.DebugPrint("multiscale_quad_retrieval(): " \ "qmaxdis[queryFrameQuad] = %s" % str(qmaxdis[queryFrameQuad])); dis_idx = np.abs(qmaxdis[queryFrameQuad] - all_max[idx]) < MAXDIS; #if False: if common.MY_DEBUG_STDOUT: """ common.DebugPrint("multiscale_quad_retrieval(): " \ "idx = %s" % str(idx)); """ common.DebugPrint("multiscale_quad_retrieval(): " \ "dis_idx = %s" % str(dis_idx)); #idx=idx(dis_idx) idx = idx[dis_idx]; #if False: if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " \ "idx (after idx = idx[dis_idx]) = %s" % str(idx)); #ori_idx=abs(qori(i)-all_ori(idx))<MAXORI; ori_idx = np.abs(qori[queryFrameQuad] - all_ori[idx]) < MAXORI; #if False: if common.MY_DEBUG_STDOUT: """ common.DebugPrint("multiscale_quad_retrieval(): " \ "all_ori = %s" % str(all_ori)); common.DebugPrint("multiscale_quad_retrieval(): " \ "qori[queryFrameQuad] = %s" % str(qori[queryFrameQuad])); """ common.DebugPrint("multiscale_quad_retrieval(): " \ "ori_idx = %s" % str(ori_idx)); #idx=idx(ori_idx); idx = idx[ori_idx]; # IMPORTANT ################################################### # IMPORTANT ################################################### # IMPORTANT ################################################### #% spatio-temporal consistency # IMPORTANT ################################################### # IMPORTANT ################################################### # IMPORTANT ################################################### #if numel(idx) > 0: if idx.size > 0: # Normally cropflag == 0 if cropflag == 0: dy = qcen[queryFrameQuad, 0] - all_cen[idx, 0]; dx = qcen[queryFrameQuad, 1] - all_cen[idx, 1]; #D=dy.^2+dx.^2; D = dy**2 + dx**2; co_idx = D < pow(st_threshold, 2); idx = idx[co_idx]; else: """ We substitute iii - 1 with iii, since we want to number arrays from 0 (not from 1 like in Matlab). """ #for iii in range(1, len(idx) + 1): for iii in range(len(idx)): #space_xy(i,(all_id(idx(iii))-RD_start)*2+1:(all_id(idx(iii))-RD_start)*2+2) = all_cen(idx(iii),:) space_xy[queryFrameQuad, \ (all_id[idx[iii]] - RD_start) * 2: (all_id[idx[iii] - 1] - RD_start) * 2 + 1] = \ all_cen[idx[iii], :] #hh=hist(all_id(idx),RD_start:RD_end); # It has to be an np.array because we multiply it with a scalar histoRange = np.array(range(RD_start, RD_end + 1)); hh = Matlab.hist(x=all_id[idx], binCenters=histoRange); #if False: if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " \ "hh = %s" % (str(hh))); common.DebugPrint("multiscale_quad_retrieval(): " \ "hh.shape = %s" % (str(hh.shape))); """ common.DebugPrint("multiscale_quad_retrieval(): " \ "all_id = %s" % (str(all_id))); """ common.DebugPrint("multiscale_quad_retrieval(): " \ "all_id.shape = %s" % (str(all_id.shape))); common.DebugPrint("multiscale_quad_retrieval(): " \ "idx = %s" % (str(idx))); common.DebugPrint("multiscale_quad_retrieval(): " \ "idx.shape = %s" % (str(idx.shape))); # % nz can be computed more optimally #nz=find(hh~=0); # nz can be computed more optimally # np.nonzero() always returns a tuple, even if it contains 1 element since hh has only 1 dimension nz = np.nonzero(hh != 0)[0]; #if False: if common.MY_DEBUG_STDOUT: common.DebugPrint("multiscale_quad_retrieval(): " \ "nz = %s" % (str(nz))); common.DebugPrint("multiscale_quad_retrieval(): " \ "nz.shape = %s" % (str(nz.shape))); #if numel(nz) > 0: if nz.size > 0: #%%----text-retrieval-like #votes(nz, tol_i) = votes(nz, tol_i) + log10(length(RD) / (length(nz)))^2 #Note: log10(a)^2 means (log10(a))^2 #PREVIOUSLY #myVal = pow(math.log10(float(len(RD)) / len(nz)), 2); myVal = pow(math.log10(float(len(r_harlocs)) / len(nz)), 2); #if False: if common.MY_DEBUG_STDOUT: """ common.DebugPrint("multiscale_quad_retrieval(): " \ "len(RD) = %d" % len(RD)); """ common.DebugPrint("multiscale_quad_retrieval(): " \ "len(r_harlocs) = %d" % len(r_harlocs)); common.DebugPrint("multiscale_quad_retrieval(): " \ "len(nz) = %d" % len(nz)); common.DebugPrint("multiscale_quad_retrieval(): " \ "myVal = %.5f" % myVal); # PREVIOUSLY votes[nz, tol_i] = votes[nz, tol_i] + myVal; # votes(nz)=votes(nz)+log10(length(RD)/(length(nz))); # votes(nz)=votes(nz)+1; #if False: if common.MY_DEBUG_STDOUT: """ common.DebugPrint("multiscale_quad_retrieval(): " \ "Votes_space.shape = %s" % (str(Votes_space.shape))); common.DebugPrint("multiscale_quad_retrieval(): " \ "votes.shape = %s" % (str(votes.shape))); """ common.DebugPrint("multiscale_quad_retrieval(): " \ "votes.shape = %s" % (str(votes.shape))); common.DebugPrint("multiscale_quad_retrieval(): " \ "votes = %s" % (str(votes))); return (queryFrame, np.ravel(votes)); # NOT performing these in each worker - the central dispatcher will do these if False: #Votes_space(:,q)=votes; # Gives: "ValueError: output operand requires a reduction, but reduction is not enabled" #Votes_space[:, queryFrame - 1] = votes; Votes_space[:, queryFrame] = np.ravel(votes); if cropflag == 0: HH[:, queryFrame] = 1; else: """ HH[:, queryFrame] = spatial_consistency.spatial_consistency(space_xy, \ qcen, len(RD), st_threshold, cropflag); """ HH[:, queryFrame] = spatial_consistency.spatial_consistency(space_xy, \ qcen, len(r_harlocs), st_threshold, cropflag);
def calculate_trigger(self): pt_cnt = self.point_count.currentIndex() if (pt_cnt == 0): raise Exception("Point count cannot be 0") self.silo.getSilo().set_point_count(pt_cnt) self.points_widget.update_silo(self.silo) repose = self.textbox_repose_angle.text() if (repose == ""): raise Exception("Must give repose angle") repose = float(repose) self.silo.getSilo().set_repose_angle(repose) error = self.textbox_percent_error.text() if (error == ""): raise Exception("Must give percent error") error = float(error) self.silo.getSilo().set_percent_error(error) # perform calculation fill_pts = [] draw_pts = [] for entry in self.silo.getSilo().get_point_pos(): if entry.isFill(): fill_pts.append([entry.getY(), entry.getX()]) else: draw_pts.append([entry.getX(), entry.getX()]) if isinstance(self.silo, Silo.SiloCircle): pile_struct = Matlab.pile( fill_pts, draw_pts, [self.silo.getRadius() * 2, self.silo.getRadius() * 2], repose) else: pile_struct = Matlab.pile( fill_pts, draw_pts, [self.silo.getSideLen(), self.silo.getSideLen()], repose) error_struct = Matlab.error(pile_struct["x"], pile_struct["y"], pile_struct["z"]) self.silo.getSilo().set_plot(pile_struct["x"], pile_struct["y"], pile_struct["z"]) self.graph.graph3d.set_mesh(pile_struct["x"], pile_struct["y"], pile_struct["z"]) self.graph.graph3d.canvas.draw() red_x = [] red_y = [] red_z = [] for row_i in range(len(error_struct["data"])): for col_i in range(len(error_struct["data"][row_i])): if (error_struct["map"][row_i, col_i] == 1): red_x.append(pile_struct["x"][row_i, col_i]) red_y.append(pile_struct["y"][row_i, col_i]) red_z.append(error_struct["data"][row_i, col_i]) self.silo.getSilo().set_min_plot(red_x, red_y, red_z) self.graph.graph3d_red.set_mesh(red_x, red_y, red_z) self.graph.graph3d_red.canvas.draw() self.silo.getSilo().set_sensor_locale(red_x, red_y) self.graph.graph_pos.set_mesh(red_x, red_y) self.graph.graph_pos.canvas.draw() error_index = [] for i in range(len(error_struct["error_list"])): error_index.append(i + error_struct["start_count"]) error_index.reverse() self.silo.getSilo().set_error_plot(error_index, error_struct["error_list"]) self.graph.graph_error.set_mesh(error_index, error_struct["error_list"]) self.graph.graph_error.canvas.draw() pass
def my_nms(cim, radius, thresh): common.DebugPrint("Entered my_nms(cim.shape=%s, radius=%s, thresh=%s)" % \ (str(cim.shape), str(radius), str(thresh))); #%% modification of Peter-Kovesi non-maximum suppression software by #%% G.Evangelidis common.DebugPrint("my_nms(): cim = %s" % str(cim)); #%subPixel = nargout == 4; % We want sub-pixel locations #subPixel=1; subPixel = 1; #[rows,cols] = size(cim) rows, cols = cim.shape; common.DebugPrint("my_nms(): rows, cols (cim.shape) = %s" % str((rows, cols))); #% Extract local maxima by performing a grey scale morphological #% dilation and then finding points in the corner strength image that #% match the dilated image and are also greater than the threshold. sze = 2 * radius + 1 #% Size of dilation mask. common.DebugPrint("my_nms(): cim.shape = %s" % str(cim.shape)); #common.DebugPrint("my_nms(): cim = %s" % str(cim)); common.DebugPrint("my_nms(): sze = %s" % str(sze)); """ Alex: we pass sze only being odd number and order = sze^2 makes ordfilt2() return the maximum from the domain. """ #%% This modification runs 4x faster (11 secs less in a 6-scale approach) #mx = ordfilt2(cim,sze^2,ones(sze)); #% Grey-scale dilate. mx = Matlab.ordfilt2(cim, pow(sze, 2), np.ones((sze, sze))); #% Grey-scale dilate. if common.MY_DEBUG_STDOUT: common.DebugPrint("my_nms(): mx = %s" % str(mx)); #% mx=my_max_filter(cim,[sze,sze]); %my mex-file for max-filter #%% This modification verify that the central point is the unique maximum in #%% neighborhood #% mx2= ordfilt2(cim,sze^2-1,ones(sze)); % This is used to vrify that the #% central point is the unique maximum in neighborhood #% imagesc(cim) #% pause #% imagesc(mx) #% pause #% close #% Make mask to exclude points within radius of the image boundary. #bordermask = zeros(size(cim)); #bordermask = np.zeros(cim.shape); bordermask = np.zeros(cim.shape, dtype=np.uint8); #Alex: sets to 1 the matrix, except the first and last radius lines and first and last radius columns, which are left 0 #bordermask(radius+1:end-radius, radius+1:end-radius) = 1; bordermask[radius: -radius, radius: -radius] = 1; #% Find maxima, threshold, and apply bordermask #cimmx = (cim==mx) & (cim>thresh) & bordermask; #%Peter-Kovesi cimmx = np.logical_and( \ np.logical_and((cim == mx), (cim > thresh)), \ bordermask); #%Peter-Kovesi #% cimmx = (cim==mx) & (cim~=mx2) & (cim>thresh) & bordermask; %my modification common.DebugPrint("my_nms(): cimmx.shape = %s" % str(cimmx.shape)); #common.DebugPrint("my_nms(): cimmx = %s" % str(cimmx)); #[r,c] = find(cimmx) #% Find row,col coords. if True: """ Retrieving indices in "Fortran" (column-major) order, like in Matlab. We do this in order to get the harris points sorted like in Matlab. """ c, r = np.nonzero(cimmx.T); else: # Retrieving indices in row-major order. r, c = np.nonzero(cimmx); common.DebugPrint("my_nms(): r = %s" % str(r)); common.DebugPrint("my_nms(): c = %s" % str(c)); if subPixel: #% Compute local maxima to sub pixel accuracy # From Matlab help: "Determine whether array is empty" "An empty array has at least one dimension of size zero, for example, 0-by-0 or 0-by-5." #if not isempty(r): #% ...if we have some ponts to work with if r.size > 0: #% ...if we have some ponts to work with #ind = sub2ind(size(cim),r,c); #% 1D indices of feature points ind = Matlab.sub2ind(cim.shape, r, c); #% 1D indices of feature points w = 1; #% Width that we look out on each side of the feature #% point to fit a local parabola if common.MY_DEBUG_STDOUT: common.DebugPrint("my_nms(): ind.shape = %s" % str(ind.shape)); common.DebugPrint("my_nms(): ind = %s" % str(ind)); common.DebugPrint("my_nms(): ind - w = %s" % str(ind - w)); # Don't forget that we are now in column major ('F'/Fortran) order #% Indices of points above, below, left and right of feature point #indrminus1 = max(ind-w,1); #indrminus1 = np.max(ind - w, 0); # In Matlab this is what it returns for 1D ind: assert ind.ndim == 1; indrminus1 = ind - w; #indrplus1 = min(ind+w,rows*cols); #indrplus1 = np.min(ind + w, rows * cols); # In Matlab this is what it returns for 1D ind: assert ind.ndim == 1; indrplus1 = ind + w; #indcminus1 = max(ind-w*rows,1); #indcminus1 = np.max(ind - w * rows, 1); # In Matlab this is what it returns for 1D ind: assert ind.ndim == 1; indcminus1 = ind - w * rows; #indcplus1 = min(ind+w*rows,rows*cols); #indcplus1 = np.min(ind + w * rows, rows * cols); # In Matlab this is what it returns for 1D ind: assert ind.ndim == 1; indcplus1 = ind + w * rows; # De-flattening the index arrays back into tuple, as accepted by numpy # See http://docs.scipy.org/doc/numpy/reference/generated/numpy.unravel_index.html ind = np.unravel_index(ind, cim.shape, order="F"); indrminus1 = np.unravel_index(indrminus1, cim.shape, order="F"); indrplus1 = np.unravel_index(indrplus1, cim.shape, order="F"); indcminus1 = np.unravel_index(indcminus1, cim.shape, order="F"); indcplus1 = np.unravel_index(indcplus1, cim.shape, order="F"); #% Solve for quadratic down rows #cy = cim(ind); cy = cim[ind]; # In Matlab ay has float elements ay = (cim[indrminus1] + cim[indrplus1]) / 2.0 - cy; #by = ay + cy - cim(indrminus1); by = ay + cy - cim[indrminus1]; #rowshift = -w*by./(2*ay); #% Maxima of quadradic rowshift = -w * by / (2.0 * ay); #% Maxima of quadradic #% Solve for quadratic across columns #cx = cim(ind); cx = cim[ind]; #ax = (cim(indcminus1) + cim(indcplus1))/2 - cx; ax = (cim[indcminus1] + cim[indcplus1]) / 2.0 - cx; #bx = ax + cx - cim(indcminus1); bx = ax + cx - cim[indcminus1]; #colshift = -w*bx./(2*ax); #% Maxima of quadratic colshift = -w * bx / (2.0 * ax); #% Maxima of quadratic rsubp = r + rowshift; #% Add subpixel corrections to original row csubp = c + colshift; #% and column coords. else: #rsubp = []; csubp = []; rsubp = np.array([]); csubp = np.array([]); """ % if nargin==4 & ~isempty(r) % Overlay corners on supplied image. % figure, imshow(im,[]), hold on % if subPixel % plot(csubp,rsubp,'r+'), title('corners detected'); % else % plot(c,r,'r+'), title('corners detected'); % end % end """ return r, c, rsubp, csubp;