def iteration_standalone_mqr(query_frame):
r_quads_tree = g.r_quadsTree
r_harlocs = g.r_harlocs
q_harlocs = g.q_harlocs
md_threshold = g.md_threshold
st_threshold = g.st_threshold
all_ori = g.all_ori
all_id = g.all_id
all_max = g.all_max
all_cen = g.all_cen
nos = g.nos
scale_index = g.scale_index
crop_flag = g.crop_flag
sequence = g.sequence
rd_start = g.rd_start
rd_end = g.rd_end
maxdis = g.maxdis
maxori = g.maxori
tolers = g.tolers
"""
We make pp reference the desired multiharloc list for the query video
frame query_frame
"""
pp = q_harlocs[query_frame]
"""
Alex: for the query frame query_frame we retrieve, for scale scale_index, the
harris features in var points.
Then we build the quads from points.
Then for each quad (4 float values) we query the corresponding scale
kd-tree, and we get the indices.
Then we build the histogram and compute idf, ....!!!!
Note: scale is 1 for original frame resolution and the higher
we go we have lower image resolutions (we go higher in the
Guassian pyramid I think).
"""
points = pp[pp[:, 2] == scale_index, 0:2]
qout, qcen, qmaxdis, qori = findquads.findquads(points, md_threshold, 0)
common.DebugPrint("multiscale_quad_retrieval(): query_frame = %d, "
"qout.shape = %s" % (query_frame, str(qout.shape)))
space_xy = np.zeros((qcen.shape[0], 2 * len(r_harlocs))) + np.nan
votes = np.zeros((len(r_harlocs), 1))
assert isinstance(tolers, float)
"""
We substitute queryFrameQuad - 1 with queryFrameQuad, since we want
to number arrays from 0 (not from 1 like in Matlab).
"""
for queryFrameQuad in range(qout.shape[0]):
"""
Matlab's polymorphism is really bugging here: although it's
normally a float, tolers is considered to be a size 1 vector...
so len(tolers) == 1
"""
"""
We substitute tol_i - 1 with tol, since we want
to number arrays from 0 (not from 1 like in Matlab).
"""
for tol_i in range(1):
tol = tolers
# default for first PAMI with tol= 0.1 approximately
# NOTE: SciPy's KDTree finds a few more results, in some cases,
# than the Matlab code from Evangelidis.
# tol is a scalar representing the radius of the ball
if config.KDTREE_IMPLEMENTATION == 0:
idx = r_quads_tree.query_ball_point(qout[queryFrameQuad, :],
tol)
elif config.KDTREE_IMPLEMENTATION == 1:
pt = qout[queryFrameQuad, :]
pt = np.array([[pt[0], pt[1], pt[2], pt[3]]], dtype=np.float32)
retval, idx, dists = r_quads_tree.radiusSearch(
query=pt,
radius=(tol ** 2),
maxResults=NUM_MAX_ELEMS,
params=search_params)
if common.MY_DEBUG_STDOUT:
common.DebugPrint(
"multiscale_quad_retrieval(): radiusSearch's retval "
"(at query_frame=%d, queryFrameQuad=%d) is %d\n" % (
query_frame, queryFrameQuad, retval))
idx = idx[0]
dists = dists[0]
idx = idx[: retval]
dists = dists[: retval]
if common.MY_DEBUG_STDOUT:
print("multiscale_quad_retrieval(): "
"qout[queryFrameQuad, :] = %s" % str(
qout[queryFrameQuad, :]))
print("multiscale_quad_retrieval(): "
"idx = %s" % str(idx))
print("multiscale_quad_retrieval(): "
"tol = %s" % str(tol))
if config.KDTREE_IMPLEMENTATION == 0:
print("multiscale_quad_retrieval(): "
"r_quads_tree.data[idx] = %s" %
str(r_quads_tree.data[idx]))
# We print the distances to the points returned in idx
a = qout[queryFrameQuad, :]
idx = np.array(idx)
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): "
"all_max.shape = %s" % str(all_max.shape))
common.DebugPrint("multiscale_quad_retrieval(): "
"qmaxdis.shape = %s" % str(qmaxdis.shape))
common.DebugPrint("multiscale_quad_retrieval(): "
"qmaxdis = %s" % str(qmaxdis))
common.DebugPrint("multiscale_quad_retrieval(): "
"qori.shape = %s" % str(qori.shape))
common.DebugPrint("multiscale_quad_retrieval(): "
"qori = %s" % str(qori))
if len(idx) == 0:
# NOT A GOOD IDEA: continue
# idx = np.array([])
dis_idx = np.array([])
ori_idx = np.array([])
else:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): "
"queryFrameQuad = %s" % str(
queryFrameQuad))
common.DebugPrint("multiscale_quad_retrieval(): "
"all_max[idx] = %s" % str(all_max[idx]))
common.DebugPrint("multiscale_quad_retrieval(): "
"qmaxdis[queryFrameQuad] = %s" % str(
qmaxdis[queryFrameQuad]))
dis_idx = np.abs(
qmaxdis[queryFrameQuad] - all_max[idx]) < maxdis
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): "
"dis_idx = %s" % str(dis_idx))
idx = idx[dis_idx]
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): "
"idx (after idx = idx[dis_idx]) = "
"%s" % str(idx))
ori_idx = np.abs(qori[queryFrameQuad] - all_ori[idx]) < maxori
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): "
"ori_idx = %s" % str(ori_idx))
idx = idx[ori_idx]
# IMPORTANT ###################################################
# IMPORTANT ###################################################
# IMPORTANT ###################################################
# spatio-temporal consistency
# IMPORTANT ###################################################
# IMPORTANT ###################################################
# IMPORTANT ###################################################
if idx.size > 0:
# Normally crop_flag == 0
if crop_flag == 0:
dy = qcen[queryFrameQuad, 0] - all_cen[idx, 0]
dx = qcen[queryFrameQuad, 1] - all_cen[idx, 1]
D = dy ** 2 + dx ** 2
co_idx = D < pow(st_threshold, 2)
idx = idx[co_idx]
else:
"""
We substitute iii - 1 with iii, since we want
to number arrays from 0 (not from 1 like in Matlab).
"""
for iii in range(len(idx)):
space_xy[queryFrameQuad,
(all_id[idx[iii]] - rd_start) * 2: (all_id[idx[
iii] - 1] - rd_start) * 2 + 1] = \
all_cen[idx[iii], :]
# It has to be an np.array because we multiply it with a scalar
histo_range = np.array(range(rd_start, rd_end + 1))
hh = Matlab.hist(x=all_id[idx], binCenters=histo_range)
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): "
"hh = %s" % (str(hh)))
common.DebugPrint("multiscale_quad_retrieval(): "
"hh.shape = %s" % (str(hh.shape)))
common.DebugPrint("multiscale_quad_retrieval(): "
"all_id.shape = %s" % (str(all_id.shape)))
common.DebugPrint("multiscale_quad_retrieval(): "
"idx = %s" % (str(idx)))
common.DebugPrint("multiscale_quad_retrieval(): "
"idx.shape = %s" % (str(idx.shape)))
# nz can be computed more optimally
# nz=find(hh~=0) # nz can be computed more optimally
# np.nonzero() always returns a tuple, even if it contains
# 1 element since hh has only 1 dimension
nz = np.nonzero(hh != 0)[0]
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): "
"nz = %s" % (str(nz)))
common.DebugPrint("multiscale_quad_retrieval(): "
"nz.shape = %s" % (str(nz.shape)))
if nz.size > 0:
my_val = pow(math.log10(float(len(r_harlocs)) / len(nz)), 2)
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): "
"len(r_harlocs) = "
"%d" % len(r_harlocs))
common.DebugPrint("multiscale_quad_retrieval(): "
"len(nz) = %d" % len(nz))
common.DebugPrint("multiscale_quad_retrieval(): "
"my_val = %.5f" % my_val)
# PREVIOUSLY
votes[nz, tol_i] = votes[nz, tol_i] + my_val
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): "
"votes.shape = %s" % (str(votes.shape)))
common.DebugPrint("multiscale_quad_retrieval(): "
"votes = %s" % (str(votes)))
return query_frame, np.ravel(votes)
def my_nms(cim, radius, thresh):
common.DebugPrint("Entered my_nms(cim.shape=%s, radius=%s, thresh=%s)" % \
(str(cim.shape), str(radius), str(thresh)))
#%% modification of Peter-Kovesi non-maximum suppression software by
#%% G.Evangelidis
common.DebugPrint("my_nms(): cim = %s" % str(cim))
#%subPixel = nargout == 4; % We want sub-pixel locations
#subPixel=1;
subPixel = 1
#[rows,cols] = size(cim)
rows, cols = cim.shape
common.DebugPrint("my_nms(): rows, cols (cim.shape) = %s" % str(
(rows, cols)))
#% Extract local maxima by performing a grey scale morphological
#% dilation and then finding points in the corner strength image that
#% match the dilated image and are also greater than the threshold.
sze = 2 * radius + 1 #% Size of dilation mask.
common.DebugPrint("my_nms(): cim.shape = %s" % str(cim.shape))
#common.DebugPrint("my_nms(): cim = %s" % str(cim));
common.DebugPrint("my_nms(): sze = %s" % str(sze))
"""
Alex: we pass sze only being odd number and order = sze^2 makes ordfilt2()
return the maximum from the domain.
"""
#%% This modification runs 4x faster (11 secs less in a 6-scale approach)
#mx = ordfilt2(cim,sze^2,ones(sze)); #% Grey-scale dilate.
mx = Matlab.ordfilt2(cim, pow(sze, 2), np.ones((sze, sze)))
#% Grey-scale dilate.
if common.MY_DEBUG_STDOUT:
common.DebugPrint("my_nms(): mx = %s" % str(mx))
#% mx=my_max_filter(cim,[sze,sze]); %my mex-file for max-filter
#%% This modification verify that the central point is the unique maximum in
#%% neighborhood
#% mx2= ordfilt2(cim,sze^2-1,ones(sze)); % This is used to vrify that the
#% central point is the unique maximum in neighborhood
#% imagesc(cim)
#% pause
#% imagesc(mx)
#% pause
#% close
#% Make mask to exclude points within radius of the image boundary.
#bordermask = zeros(size(cim));
#bordermask = np.zeros(cim.shape);
bordermask = np.zeros(cim.shape, dtype=np.uint8)
#Alex: sets to 1 the matrix, except the first and last radius lines and first and last radius columns, which are left 0
#bordermask(radius+1:end-radius, radius+1:end-radius) = 1;
bordermask[radius:-radius, radius:-radius] = 1
#% Find maxima, threshold, and apply bordermask
#cimmx = (cim==mx) & (cim>thresh) & bordermask; #%Peter-Kovesi
cimmx = np.logical_and( \
np.logical_and((cim == mx), (cim > thresh)), \
bordermask)
#%Peter-Kovesi
#% cimmx = (cim==mx) & (cim~=mx2) & (cim>thresh) & bordermask; %my modification
common.DebugPrint("my_nms(): cimmx.shape = %s" % str(cimmx.shape))
#common.DebugPrint("my_nms(): cimmx = %s" % str(cimmx));
#[r,c] = find(cimmx) #% Find row,col coords.
if True:
"""
Retrieving indices in "Fortran" (column-major) order, like in Matlab.
We do this in order to get the harris points sorted like in Matlab.
"""
c, r = np.nonzero(cimmx.T)
else:
# Retrieving indices in row-major order.
r, c = np.nonzero(cimmx)
common.DebugPrint("my_nms(): r = %s" % str(r))
common.DebugPrint("my_nms(): c = %s" % str(c))
if subPixel: #% Compute local maxima to sub pixel accuracy
# From Matlab help: "Determine whether array is empty" "An empty array has at least one dimension of size zero, for example, 0-by-0 or 0-by-5."
#if not isempty(r): #% ...if we have some ponts to work with
if r.size > 0: #% ...if we have some ponts to work with
#ind = sub2ind(size(cim),r,c); #% 1D indices of feature points
ind = Matlab.sub2ind(cim.shape, r, c)
#% 1D indices of feature points
w = 1
#% Width that we look out on each side of the feature
#% point to fit a local parabola
if common.MY_DEBUG_STDOUT:
common.DebugPrint("my_nms(): ind.shape = %s" % str(ind.shape))
common.DebugPrint("my_nms(): ind = %s" % str(ind))
common.DebugPrint("my_nms(): ind - w = %s" % str(ind - w))
# Don't forget that we are now in column major ('F'/Fortran) order
#% Indices of points above, below, left and right of feature point
#indrminus1 = max(ind-w,1);
#indrminus1 = np.max(ind - w, 0);
# In Matlab this is what it returns for 1D ind:
assert ind.ndim == 1
indrminus1 = ind - w
#indrplus1 = min(ind+w,rows*cols);
#indrplus1 = np.min(ind + w, rows * cols);
# In Matlab this is what it returns for 1D ind:
assert ind.ndim == 1
indrplus1 = ind + w
#indcminus1 = max(ind-w*rows,1);
#indcminus1 = np.max(ind - w * rows, 1);
# In Matlab this is what it returns for 1D ind:
assert ind.ndim == 1
indcminus1 = ind - w * rows
#indcplus1 = min(ind+w*rows,rows*cols);
#indcplus1 = np.min(ind + w * rows, rows * cols);
# In Matlab this is what it returns for 1D ind:
assert ind.ndim == 1
indcplus1 = ind + w * rows
# De-flattening the index arrays back into tuple, as accepted by numpy
# See http://docs.scipy.org/doc/numpy/reference/generated/numpy.unravel_index.html
ind = np.unravel_index(ind, cim.shape, order="F")
indrminus1 = np.unravel_index(indrminus1, cim.shape, order="F")
indrplus1 = np.unravel_index(indrplus1, cim.shape, order="F")
indcminus1 = np.unravel_index(indcminus1, cim.shape, order="F")
indcplus1 = np.unravel_index(indcplus1, cim.shape, order="F")
#% Solve for quadratic down rows
#cy = cim(ind);
cy = cim[ind]
# In Matlab ay has float elements
ay = (cim[indrminus1] + cim[indrplus1]) / 2.0 - cy
#by = ay + cy - cim(indrminus1);
by = ay + cy - cim[indrminus1]
#rowshift = -w*by./(2*ay); #% Maxima of quadradic
rowshift = -w * by / (2.0 * ay)
#% Maxima of quadradic
#% Solve for quadratic across columns
#cx = cim(ind);
cx = cim[ind]
#ax = (cim(indcminus1) + cim(indcplus1))/2 - cx;
ax = (cim[indcminus1] + cim[indcplus1]) / 2.0 - cx
#bx = ax + cx - cim(indcminus1);
bx = ax + cx - cim[indcminus1]
#colshift = -w*bx./(2*ax); #% Maxima of quadratic
colshift = -w * bx / (2.0 * ax)
#% Maxima of quadratic
rsubp = r + rowshift
#% Add subpixel corrections to original row
csubp = c + colshift
#% and column coords.
else:
#rsubp = []; csubp = [];
rsubp = np.array([])
csubp = np.array([])
"""
% if nargin==4 & ~isempty(r) % Overlay corners on supplied image.
% figure, imshow(im,[]), hold on
% if subPixel
% plot(csubp,rsubp,'r+'), title('corners detected');
% else
% plot(c,r,'r+'), title('corners detected');
% end
% end
"""
return r, c, rsubp, csubp
def IterationStandaloneMQR(queryFrame):
r_quadsTree = g.r_quadsTree
r_harlocs = g.r_harlocs
q_harlocs = g.q_harlocs
md_threshold = g.md_threshold
st_threshold = g.st_threshold
all_ori = g.all_ori
all_id = g.all_id
all_max = g.all_max
all_cen = g.all_cen
nos = g.nos
scale_index = g.scale_index
cropflag = g.cropflag
sequence = g.sequence
RD_start = g.RD_start
RD_end = g.RD_end
MAXDIS = g.MAXDIS
MAXORI = g.MAXORI
tolers = g.tolers
"""
common.DebugPrint( \
"Entered IterationStandaloneMQR(): crossref=%s, captureQ=%s, "\
"captureR=%s, refined_crossref=%s, warp_p=%s, "
"x0=%s, y0=%s, start=%s, t=%d, iWhile=%d." % \
(str(crossref), str(captureQ), str(captureR), \
str(g.refined_crossref), str(g.warp_p), \
str(g.x0), str(g.y0), str(g.start), g.t, iWhile));
common.DebugPrint("IterationStandalone(): id(g)=%s" % str(id(g)));
"""
# tic
"""
str1=['load ' q_path QD(q).name]
eval(str1)
"""
"""
We make pp reference the desired multiharloc list for the query video
frame queryFrame
"""
pp = q_harlocs[queryFrame]
#pp = np.array(pp);
#common.DebugPrint("multiscale_quad_retrieval(): pp = %s" % str(pp));
"""
Alex: for the query frame queryFrame we retrieve, for scale scale_index, the
harris features in var points.
Then we build the quads from points.
Then for each quad (4 float values) we query the corresponding scale
kd-tree, and we get the indices.
Then we build the histogram and compute idf, ....!!!!
Note: scale is 1 for original frame resolution and the higher
we go we have lower image resolutions (we go higher in the
Guassian pyramid I think).
"""
#[qout,qcen,qmaxdis,qori]=findquads(pp(pp(:,3)==scale_index,1:2),md_threshold,0);
points = pp[pp[:, 2] == scale_index, 0:2]
qout, qcen, qmaxdis, qori = findquads.findquads(points, md_threshold, 0)
common.DebugPrint("multiscale_quad_retrieval(): queryFrame = %d, " \
"qout.shape = %s" % (queryFrame, str(qout.shape)))
# disp([num2str(q) ' of ' num2str(length(QD)) ' -> ' num2str(size(qout,1)) ' quads'])
#space_xy=zeros(size(qcen,1),2*length(RD))+nan;
#space_xy = np.zeros( (qcen.shape[0], 2 * len(RD)) ) + np.nan;
space_xy = np.zeros((qcen.shape[0], 2 * len(r_harlocs))) + np.nan
# votes=zeros(length(RD),1)
#votes=zeros(length(RD),length(tolers));
#votes = np.zeros( (len(RD), 1) );
votes = np.zeros((len(r_harlocs), 1))
#nep = np.array([]);
#m_points = np.array([]);
assert isinstance(tolers, float)
"""
We substitute queryFrameQuad - 1 with queryFrameQuad, since we want
to number arrays from 0 (not from 1 like in Matlab).
"""
#for queryFrameQuad in range(1, qout.shape[0] + 1):
for queryFrameQuad in range(qout.shape[0]):
"""
Matlab's polymorphism is really bugging here: although it's
normally a float, tolers is considered to be a size 1 vector...
so len(tolers) == 1
"""
#for tol_i in range(1, len(tolers) + 1):
# tol = tolers[tol_i - 1]
"""
We substitute tol_i - 1 with tol, since we want
to number arrays from 0 (not from 1 like in Matlab).
"""
#for tol_i in range(1, 1 + 1):
for tol_i in range(1):
tol = tolers
#common.DebugPrint("multiscale_quad_retrieval(): qout[i - 1, :] = %s" % str(qout[i - 1, :]))
#% default for first PAMI with tol= 0.1 approximately
# NOTE: SciPy's KDTree finds a few more results, in some cases,
# than the Matlab code from Evangelidis.
#idx, di = kdtree_ball_query(tree, qout(i, :), tol)
#idx, distKD = kdtree_ball_query(tree, qout[i - 1, :], tol)
#idx, di = tree.query(x=xQuery, k=4)
#resPoints = [data[i] for i in resBallIndices]
# tol is a scalar representing the radius of the ball
if config.KDTREE_IMPLEMENTATION == 0:
idx = r_quadsTree.query_ball_point(qout[queryFrameQuad, :],
tol)
elif config.KDTREE_IMPLEMENTATION == 1:
#pt = qout[queryFrameQuad - 1, :].astype(np.float32);
pt = qout[queryFrameQuad, :]
pt = np.array([[pt[0], pt[1], pt[2], pt[3]]], dtype=np.float32)
retval, idx, dists = r_quadsTree.radiusSearch( \
query=pt, \
radius=(tol**2), \
maxResults=NUM_MAX_ELEMS, \
params=search_params)
if common.MY_DEBUG_STDOUT and DBGPRINT:
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"retval (number NNs) = %s" % str(retval));
"""
common.DebugPrint( \
"multiscale_quad_retrieval(): radiusSearch's retval " \
"(at queryFrame=%d, queryFrameQuad=%d) is %d\n" % (queryFrame, queryFrameQuad, retval))
idx = idx[0]
dists = dists[0]
idx = idx[:retval]
dists = dists[:retval]
if common.MY_DEBUG_STDOUT and DBGPRINT:
print("multiscale_quad_retrieval(): " \
"qout[queryFrameQuad, :] = %s" % str(qout[queryFrameQuad, :]))
print("multiscale_quad_retrieval(): " \
"idx = %s" % str(idx))
print("multiscale_quad_retrieval(): " \
"tol = %s" % str(tol))
if config.KDTREE_IMPLEMENTATION == 0:
print("multiscale_quad_retrieval(): " \
"r_quadsTree.data[idx] = %s" % \
str(r_quadsTree.data[idx]))
# We print the distances to the points returned in idx
a = qout[queryFrameQuad, :]
if False: #!!!! This is just for debugging purposes
for myI, index in enumerate(idx):
b = r_quadsTree.data[index]
"""
if False:
common.DebugPrint("multiscale_quad_retrieval(): distance to " \
"%d point (%s) inside ball = %.4f" % \
(myI, str(b), npla.norm(a - b)));
"""
idx = np.array(idx)
#if False:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): " \
"all_max.shape = %s" % str(all_max.shape))
common.DebugPrint("multiscale_quad_retrieval(): " \
"qmaxdis.shape = %s" % str(qmaxdis.shape))
common.DebugPrint("multiscale_quad_retrieval(): " \
"qmaxdis = %s" % str(qmaxdis))
common.DebugPrint("multiscale_quad_retrieval(): " \
"qori.shape = %s" % str(qori.shape))
common.DebugPrint("multiscale_quad_retrieval(): " \
"qori = %s" % str(qori))
#dis_idx=abs(qmaxdis(i)-all_max(idx))<MAXDIS;
if len(idx) == 0:
# NOT A GOOD IDEA: continue;
#idx = np.array([]);
dis_idx = np.array([])
ori_idx = np.array([])
else:
#if False:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): " \
"queryFrameQuad = %s" % str(queryFrameQuad))
common.DebugPrint("multiscale_quad_retrieval(): " \
"all_max[idx] = %s" % str(all_max[idx]))
common.DebugPrint("multiscale_quad_retrieval(): " \
"qmaxdis[queryFrameQuad] = %s" % str(qmaxdis[queryFrameQuad]))
dis_idx = np.abs(qmaxdis[queryFrameQuad] -
all_max[idx]) < MAXDIS
#if False:
if common.MY_DEBUG_STDOUT:
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"idx = %s" % str(idx));
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"dis_idx = %s" % str(dis_idx))
#idx=idx(dis_idx)
idx = idx[dis_idx]
#if False:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): " \
"idx (after idx = idx[dis_idx]) = %s" % str(idx))
#ori_idx=abs(qori(i)-all_ori(idx))<MAXORI;
ori_idx = np.abs(qori[queryFrameQuad] - all_ori[idx]) < MAXORI
#if False:
if common.MY_DEBUG_STDOUT:
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"all_ori = %s" % str(all_ori));
common.DebugPrint("multiscale_quad_retrieval(): " \
"qori[queryFrameQuad] = %s" % str(qori[queryFrameQuad]));
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"ori_idx = %s" % str(ori_idx))
#idx=idx(ori_idx);
idx = idx[ori_idx]
# IMPORTANT ###################################################
# IMPORTANT ###################################################
# IMPORTANT ###################################################
#% spatio-temporal consistency
# IMPORTANT ###################################################
# IMPORTANT ###################################################
# IMPORTANT ###################################################
#if numel(idx) > 0:
if idx.size > 0:
# Normally cropflag == 0
if cropflag == 0:
dy = qcen[queryFrameQuad, 0] - all_cen[idx, 0]
dx = qcen[queryFrameQuad, 1] - all_cen[idx, 1]
#D=dy.^2+dx.^2;
D = dy**2 + dx**2
co_idx = D < pow(st_threshold, 2)
idx = idx[co_idx]
else:
"""
We substitute iii - 1 with iii, since we want
to number arrays from 0 (not from 1 like in Matlab).
"""
#for iii in range(1, len(idx) + 1):
for iii in range(len(idx)):
#space_xy(i,(all_id(idx(iii))-RD_start)*2+1:(all_id(idx(iii))-RD_start)*2+2) = all_cen(idx(iii),:)
space_xy[queryFrameQuad, \
(all_id[idx[iii]] - RD_start) * 2: (all_id[idx[iii] - 1] - RD_start) * 2 + 1] = \
all_cen[idx[iii], :]
#hh=hist(all_id(idx),RD_start:RD_end);
# It has to be an np.array because we multiply it with a scalar
histoRange = np.array(range(RD_start, RD_end + 1))
hh = Matlab.hist(x=all_id[idx], binCenters=histoRange)
#if False:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): " \
"hh = %s" % (str(hh)))
common.DebugPrint("multiscale_quad_retrieval(): " \
"hh.shape = %s" % (str(hh.shape)))
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"all_id = %s" % (str(all_id)));
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"all_id.shape = %s" % (str(all_id.shape)))
common.DebugPrint("multiscale_quad_retrieval(): " \
"idx = %s" % (str(idx)))
common.DebugPrint("multiscale_quad_retrieval(): " \
"idx.shape = %s" % (str(idx.shape)))
# % nz can be computed more optimally
#nz=find(hh~=0); # nz can be computed more optimally
# np.nonzero() always returns a tuple, even if it contains 1 element since hh has only 1 dimension
nz = np.nonzero(hh != 0)[0]
#if False:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): " \
"nz = %s" % (str(nz)))
common.DebugPrint("multiscale_quad_retrieval(): " \
"nz.shape = %s" % (str(nz.shape)))
#if numel(nz) > 0:
if nz.size > 0:
#%%----text-retrieval-like
#votes(nz, tol_i) = votes(nz, tol_i) + log10(length(RD) / (length(nz)))^2 #Note: log10(a)^2 means (log10(a))^2 #PREVIOUSLY
#myVal = pow(math.log10(float(len(RD)) / len(nz)), 2);
myVal = pow(math.log10(float(len(r_harlocs)) / len(nz)), 2)
#if False:
if common.MY_DEBUG_STDOUT:
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"len(RD) = %d" % len(RD));
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"len(r_harlocs) = %d" % len(r_harlocs))
common.DebugPrint("multiscale_quad_retrieval(): " \
"len(nz) = %d" % len(nz))
common.DebugPrint("multiscale_quad_retrieval(): " \
"myVal = %.5f" % myVal)
# PREVIOUSLY
votes[nz, tol_i] = votes[nz, tol_i] + myVal
# votes(nz)=votes(nz)+log10(length(RD)/(length(nz)));
# votes(nz)=votes(nz)+1;
#if False:
if common.MY_DEBUG_STDOUT:
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"Votes_space.shape = %s" % (str(Votes_space.shape)));
common.DebugPrint("multiscale_quad_retrieval(): " \
"votes.shape = %s" % (str(votes.shape)));
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"votes.shape = %s" % (str(votes.shape)))
common.DebugPrint("multiscale_quad_retrieval(): " \
"votes = %s" % (str(votes)))
return (queryFrame, np.ravel(votes))
# NOT performing these in each worker - the central dispatcher will do these
if False:
#Votes_space(:,q)=votes;
# Gives: "ValueError: output operand requires a reduction, but reduction is not enabled"
#Votes_space[:, queryFrame - 1] = votes;
Votes_space[:, queryFrame] = np.ravel(votes)
if cropflag == 0:
HH[:, queryFrame] = 1
else:
"""
HH[:, queryFrame] = spatial_consistency.spatial_consistency(space_xy, \
qcen, len(RD), st_threshold, cropflag);
"""
HH[:, queryFrame] = spatial_consistency.spatial_consistency(space_xy, \
qcen, len(r_harlocs), st_threshold, cropflag)
def multiscale_quad_retrieval(r_quads_tree, r_harlocs, q_harlocs, md_threshold,
st_threshold, all_ori, all_id, all_max, all_cen,
nos, scale_index, crop_flag, sequence):
common.DebugPrint("Entered multiscale_quad_retrieval(): "
"md_threshold = %s, st_threshold = %s." %
(str(md_threshold), str(st_threshold)))
assert len(r_harlocs) != 0
assert len(q_harlocs) != 0
try:
votes_space = np.load("votes_space%d.npz" % scale_index)['arr_0']
HH = np.load("HH%d.npz" % scale_index)['arr_0']
return votes_space, HH
except:
common.DebugPrintErrorTrace()
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): r_quads_tree = %s" %
str(r_quads_tree))
common.DebugPrint(
"multiscale_quad_retrieval(): len(r_harlocs) = %d" % len(r_harlocs))
common.DebugPrint(
"multiscale_quad_retrieval(): r_harlocs = %s" % str(r_harlocs))
common.DebugPrint(
"multiscale_quad_retrieval(): q_harlocs = %s" % str(q_harlocs))
common.DebugPrint(
"multiscale_quad_retrieval(): md_threshold = %s" % str(
md_threshold))
print("multiscale_quad_retrieval(): st_threshold = %s" % str(
st_threshold))
common.DebugPrint(
"multiscale_quad_retrieval(): all_id = %s" % str(all_id))
common.DebugPrint("multiscale_quad_retrieval(): all_id.shape = %s" % (
str(all_id.shape)))
common.DebugPrint(
"multiscale_quad_retrieval(): sequence = %s" % str(sequence))
print("multiscale_quad_retrieval(): crop_flag = %s" % str(crop_flag))
t1 = float(cv2.getTickCount())
if scale_index > nos:
assert scale_index <= nos
# TODO: take out rd_start
rd_start = 0
rd_end = len(r_harlocs) - 1
j = 1
"""
Inspired from
https://stackoverflow.com/questions/17559140/matlab-twice-as-fast-as-numpy
BUT doesn't help in this case:
votes_space = np.asfortranarray(np.zeros( (len(RD), len(QD)) ))
"""
votes_space = np.zeros((len(r_harlocs), len(q_harlocs)))
# Make a distinct copy of HH from votes_space...
# TODO: use MAYBE even np.bool - OR take it out
HH = np.zeros((len(r_harlocs), len(q_harlocs)), dtype=np.int8)
# it helps to make more strict the threshold as the scale goes up
tolers = 0.1 - float(scale_index) / 100.0
maxdis = 3 + scale_index
maxori = 0.25
# TODO: I am using multiprocessing.Poll and return votes the dispatcher
# assembles the results, but the results are NOT the same with the serial
# case - although they look pretty decent, but they seem to be
# suboptimal - dp_alex returns suboptimal cost path for
# USE_MULTITHREADING == True instead of False.
# (Note: running under the same preconditions
# multiscale_quad_retrieval I got the same results in dp_alex().
"""
if False: #config.USE_MULTITHREADING == True:
global g
g.r_quads_tree = r_quads_tree
g.r_harlocs = r_harlocs
g.q_harlocs = q_harlocs
g.md_threshold = md_threshold
g.st_threshold = st_threshold
g.all_ori = all_ori
g.all_id = all_id
g.all_max = all_max
g.all_cen = all_cen
g.nos = nos
g.scale_index = scale_index
g.crop_flag = crop_flag
g.sequence = sequence
g.RD_start = RD_start
g.RD_end = RD_end
g.maxdis = maxdis
g.maxori = maxori
g.tolers = tolers
#Start worker processes to use on multi-core processor (able to run
# in parallel - no GIL issue if each core has it's own VM)
pool = multiprocessing.Pool(processes=config.numProcesses)
print("multiscale_quad_retrieval(): Spawned a pool of %d workers" %
config.numProcesses)
listParams = range(0, len(q_harlocs)) #!!!!TODO: use counterStep, config.initFrame[indexVideo]
#res = pool.map(iteration_standalone_mqr, listParams)
# See https://docs.python.org/2/library/multiprocessing.html#module-multiprocessing.pool
res = pool.map(func=iteration_standalone_mqr, iterable=listParams,
chunksize=1)
print("Pool.map returns %s" % str(res)) #x0.size + 1
# From https://medium.com/building-things-on-the-internet/40e9b2b36148
# close the pool and wait for the work to finish
pool.close()
pool.join()
# Doing the "reduce" phase after the workers have finished :)
assert len(res) == len(q_harlocs)
for query_frame, resE in enumerate(res):
resEIndex = resE[0]
resE = resE[1]
assert resEIndex == query_frame
# Gives: "ValueError: output operand requires a reduction, but reduction is not enabled"
#votes_space[:, query_frame - 1] = votes
votes_space[:, query_frame] = resE
for query_frame in range(len(q_harlocs)):
if crop_flag == 0:
HH[:, query_frame] = 1
else:
HH[:, query_frame] = spatial_consistency.spatial_consistency(space_xy,
qcen, len(r_harlocs), st_threshold, crop_flag)
try:
np.savez_compressed("votes_space%d" % scale_index, votes_space)
np.savez_compressed("HH%d" % scale_index, HH)
except:
common.DebugPrintErrorTrace()
return votes_space, HH
"""
"""
We substitute q - 1 with q, since we want
to number arrays from 0 (not from 1 like in Matlab).
"""
for query_frame in range(len(q_harlocs)):
common.DebugPrint("multiscale_quad_retrieval(): Starting iteration "
"query_frame = %d" % query_frame)
"""
We make pp reference the desired multiharloc list for the query video
frame query_frame
"""
pp = q_harlocs[query_frame]
points = pp[pp[:, 2] == scale_index, 0:2]
qout, qcen, qmaxdis, qori = findquads.findquads(points, md_threshold, 0)
if common.MY_DEBUG_STDOUT:
print("multiscale_quad_retrieval(): query_frame = %d, "
"qout.shape (number of quads for query frame query_frame) = "
"%s" % (query_frame, str(qout.shape)))
space_xy = np.zeros((qcen.shape[0], 2 * len(r_harlocs))) + np.nan
votes = np.zeros((len(r_harlocs), 1))
assert isinstance(tolers, float)
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): quads of query "
"frame %d are: " % query_frame)
common.DebugPrint(" qout = %s" % str(qout))
"""
Alex: for each quad (4 floats) of the query frame from Harris feature of
scale scale_index
Note: all_id stores the reference frame id for each quad descriptor.
"""
"""
We substitute queryFrameQuad - 1 with queryFrameQuad, since we want
to number arrays from 0 (not from 1 like in Matlab).
"""
for queryFrameQuad in range(qout.shape[0]):
common.DebugPrint("multiscale_quad_retrieval(): Starting iteration "
"queryFrameQuad = %d" % queryFrameQuad)
"""
Matlab's polymorphism is really bugging here: although it's
normally a float, tolers is considered to be a size 1 vector...
so len(tolers) == 1
"""
"""
We substitute tol_i - 1 with tol, since we want
to number arrays from 0 (not from 1 like in Matlab).
"""
for tol_i in range(1):
tol = tolers
# default for first PAMI with tol= 0.1 approximately
# NOTE: SciPy's KDTree finds a few more results, in some cases,
# than the Matlab code from Evangelidis.
# tol is a scalar representing the radius of the ball
if config.KDTREE_IMPLEMENTATION == 0:
idx = r_quads_tree.query_ball_point(qout[queryFrameQuad, :],
tol)
elif config.KDTREE_IMPLEMENTATION == 1:
pt = qout[queryFrameQuad, :]
pt = np.array([[pt[0], pt[1], pt[2], pt[3]]],
dtype=np.float32)
retval, idx, dists = r_quads_tree.radiusSearch(
query=pt,
radius=(tol ** 2),
maxResults=NUM_MAX_ELEMS,
params=search_params)
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): "
"radiusSearch's retval (at "
"query_frame=%d, queryFrameQuad=%d) "
"is %d" %
(query_frame, queryFrameQuad, retval))
idx = idx[0]
dists = dists[0]
"""
Note: retval is the number of neighbors returned from the
radiusSearch().
But the idx and the dists can have more elements than the
returned retval.
"""
idx = idx[: retval]
dists = dists[: retval]
if common.MY_DEBUG_STDOUT:
print("multiscale_quad_retrieval(): "
"qout[queryFrameQuad, :] = %s" %
str(qout[queryFrameQuad, :]))
print("multiscale_quad_retrieval(): "
"idx = %s" % str(idx))
print("multiscale_quad_retrieval(): "
"dists = %s" % str(dists))
print("multiscale_quad_retrieval(): "
"tol = %s" % str(tol))
if config.KDTREE_IMPLEMENTATION == 0:
print("multiscale_quad_retrieval(): "
"r_quads_tree.data[idx] = %s" %
str(r_quads_tree.data[idx]))
if common.MY_DEBUG_STDOUT:
a = qout[queryFrameQuad, :]
if config.KDTREE_IMPLEMENTATION == 0:
for myI, index in enumerate(idx):
b = r_quads_tree.data[index]
else:
pass
idx = np.array(idx)
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): "
"all_max.shape = %s" % str(all_max.shape))
common.DebugPrint("multiscale_quad_retrieval(): "
"qmaxdis.shape = %s" % str(qmaxdis.shape))
common.DebugPrint("multiscale_quad_retrieval(): "
"qmaxdis = %s" % str(qmaxdis))
common.DebugPrint("multiscale_quad_retrieval(): "
"qori.shape = %s" % str(qori.shape))
common.DebugPrint("multiscale_quad_retrieval(): "
"qori = %s" % str(qori))
if len(idx) == 0:
# NOT A GOOD IDEA: continue
dis_idx = np.array([])
ori_idx = np.array([])
else:
if common.MY_DEBUG_STDOUT:
print("multiscale_quad_retrieval(): "
"queryFrameQuad = %s" % str(queryFrameQuad))
print("multiscale_quad_retrieval(): "
"all_max[idx] = %s" % str(all_max[idx]))
print("multiscale_quad_retrieval(): "
"qmaxdis[queryFrameQuad] = %s" %
str(qmaxdis[queryFrameQuad]))
if USE_GPS_COORDINATES:
# We look only at a part of the reference video
"""
Since in some cases the video temporal alignment is
difficult to do due to similar portions in the
trajectory (see the drone videos, clip 3_some_lake)
we "guide" the temporal alignment by restricting
the reference frame search space - this is useful
when we have the geolocation (GPS) coordinate for
each frame.
"""
if common.MY_DEBUG_STDOUT:
print("multiscale_quad_retrieval(): "
"all_id = %s" % str(all_id))
if all_id.ndim == 2:
# TODO: put this at the beginning of the
# function
assert all_id.shape[1] == 1
"""
We flatten the array all_id
Note: We don't use order="F" since it's
basically 1-D array
"""
all_id = np.ravel(all_id)
# TODO: put start and end frame in config - or compute
# it from geolocation
sub_idx = np.logical_and((all_id[idx] >= 2030 - 928),
(all_id[idx] <= 2400 - 928))
idx = idx[sub_idx]
if common.MY_DEBUG_STDOUT:
print("multiscale_quad_retrieval(): "
"all_id = %s" % str(all_id))
print("multiscale_quad_retrieval(): "
"sub_idx = %s" % str(sub_idx))
print("multiscale_quad_retrieval(): "
"idx = %s" % str(idx))
if FILTER:
dis_idx = np.abs(
qmaxdis[queryFrameQuad] - all_max[idx]) < maxdis
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): "
"dis_idx = %s" % str(dis_idx))
idx = idx[dis_idx]
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): "
"idx (after idx = idx[dis_idx]) = "
"%s" % str(idx))
if FILTER:
ori_idx = np.abs(
qori[queryFrameQuad] - all_ori[idx]) < maxori
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): "
"ori_idx = %s" % str(ori_idx))
idx = idx[ori_idx]
# IMPORTANT ###################################################
# IMPORTANT ###################################################
# IMPORTANT ###################################################
# spatio-temporal consistency
# IMPORTANT ###################################################
# IMPORTANT ###################################################
# IMPORTANT ###################################################
if idx.size > 0:
if crop_flag == 0:
if FILTER:
"""
Alex: this is a simple procedure of eliminating
False Positive (FP) matches, as presented in
Section 4.2 of TPAMI 2013 paper.
Basically it filters out quad matches that have
centroids st_threshold away from the query quad.
Note: all_cen are the controids of all reference
quads.
"""
dy = qcen[queryFrameQuad, 0] - all_cen[idx, 0]
dx = qcen[queryFrameQuad, 1] - all_cen[idx, 1]
D = dy ** 2 + dx ** 2
co_idx = D < pow(st_threshold, 2)
idx = idx[co_idx]
else:
"""
We substitute iii - 1 with iii, since we want
to number arrays from 0 (not from 1 like in Matlab).
"""
for iii in range(len(idx)):
space_xy[queryFrameQuad,
(all_id[idx[iii]] - rd_start) * 2: (all_id[idx[
iii] - 1] - rd_start) * 2 + 1] = \
all_cen[idx[iii], :]
# It has to be an np.array because we multiply it with a
# scalar
histo_range = np.array(range(rd_start, rd_end + 1))
hh = Matlab.hist(x=all_id[idx], binCenters=histo_range)
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): "
"hh = %s" % (str(hh)))
common.DebugPrint("multiscale_quad_retrieval(): "
"hh.shape = %s" % (str(hh.shape)))
common.DebugPrint("multiscale_quad_retrieval(): "
"all_id.shape = %s" % (
str(all_id.shape)))
common.DebugPrint("multiscale_quad_retrieval(): "
"idx = %s" % (str(idx)))
common.DebugPrint("multiscale_quad_retrieval(): "
"idx.shape = %s" % (str(idx.shape)))
# % nz can be computed more optimally
nz = np.nonzero(hh != 0)[0]
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): "
"nz = %s" % (str(nz)))
common.DebugPrint("multiscale_quad_retrieval(): "
"nz.shape = %s" % (str(nz.shape)))
if nz.size > 0:
my_val = pow(
math.log10(float(len(r_harlocs)) / len(nz)), 2)
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): "
"len(r_harlocs) = %d" % len(
r_harlocs))
common.DebugPrint("multiscale_quad_retrieval(): "
"len(nz) = %d" % len(nz))
common.DebugPrint("multiscale_quad_retrieval(): "
"my_val = %.5f" % my_val)
# PREVIOUSLY
votes[nz, tol_i] = votes[nz, tol_i] + my_val
if common.MY_DEBUG_STDOUT:
print("multiscale_quad_retrieval(): "
"votes.shape = %s" % (str(votes.shape)))
if (np.abs(votes) < 1.0e-10).all():
print("multiscale_quad_retrieval(): votes = 0 (all zeros)")
else:
print("multiscale_quad_retrieval(): votes = %s" % (str(votes)))
# Note: since votes is basically a 1-D vector, we don't use the
# Fortran order
votes_space[:, query_frame] = np.ravel(votes)
if crop_flag == 0:
HH[:, query_frame] = 1
else:
HH[:, query_frame] = spatial_consistency.spatial_consistency(
space_xy,
qcen, len(r_harlocs), st_threshold, crop_flag)
if common.MY_DEBUG_STDOUT:
print("multiscale_quad_retrieval(scale_index=%d): "
"votes_space =\n%s" % (scale_index, str(votes_space)))
try:
np.savez_compressed("votes_space%d" % scale_index, votes_space)
np.savez_compressed("HH%d" % scale_index, HH)
except:
common.DebugPrintErrorTrace()
t2 = float(cv2.getTickCount())
my_time = (t2 - t1) / cv2.getTickFrequency()
print("multiscale_quad_retrieval() took %.6f [sec]" % my_time)
return votes_space, HH
def multi_scale_harris_Evangelidis(im, nos, disp):
common.DebugPrint("Entered multi_scale_harris_Evangelidis(nos=%d, disp=%s)" % \
(nos, str(disp)));
#tic
#if size(im, 3) == 3:
# im = rgb2gray(im)
tMSH1 = float(cv2.getTickCount());
if im.ndim == 3:
im = common.ConvertImgToGrayscale(im);
#im = im2double(im)
# From https://stackoverflow.com/questions/10873824/how-to-convert-2d-float-numpy-array-to-2d-int-numpy-array:
#DO NOT USE: im = im.astype(np.uint8); # This messes up tremendously the computation of harlocs
#im = im.astype(float);
im = im.astype(np.float32);
#!!!!COR is an UNused variable
#COR = zeros(size(im,1), size(im,2), size(im,3))
#COR = np.zeros((im.shape[0], im.shape[1], im.shape[1]));
# scale values
sigma_0 = 1.2;
#n=[0:nos-1]; %scale levels
n = np.array(range(nos)); #scale levels
#sigma_D=sqrt(1.8).^n*sigma_0
sigma_D = math.sqrt(1.8) ** n * sigma_0;
#points=[];
points = np.array([]);
#scn=sigma_D.^4;
scn = sigma_D ** 4;
#for i=1:length(sigma_D)
#for i in range(1, len(sigma_D) + 1):
for i in range(1, nos + 1):
common.DebugPrint("multi_scale_harris_Evangelidis(): i = %s" % (str(i)));
#sd=sigma_D(i); %differentiation (local) scale
sd = sigma_D[i - 1]; #%differentiation (local) scale
#si=sd/.5; %integration scale
si = sd / 0.5; #integration scale
w = 3 * sd; # size for gaussian kernel to compute derivatives: 3 times local_scale
r_w = int(round(w));
common.DebugPrint("multi_scale_harris_Evangelidis(): r_w = %s" % \
(str(r_w)));
#if mod(round(w),2):
if (r_w % 2) == 1:
#[xco,yco] = meshgrid(-round(w):round(w),-round(w):round(w))
mRange = range(-r_w, r_w + 1);
xco, yco = Matlab.meshgrid(mRange, mRange);
else:
#[xco,yco] = meshgrid(-(round(w)+1):round(w)+1,-(round(w)+1):round(w)+1)
mRange = range(-(r_w + 1), r_w + 2);
xco, yco = Matlab.meshgrid(mRange, mRange);
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): xco = %s" % \
str(xco));
common.DebugPrint("multi_scale_harris_Evangelidis(): yco = %s" % \
str(yco));
# Note: even for HD frames, xco.shape = (11, 11) (yco the same)
#arg = -(xco.*xco + yco.*yco) / (2*sd*sd)
#arg = -(xco * xco + yco * yco) / (2.0 * sd * sd);
arg = -(xco ** 2 + yco ** 2) / (2.0 * sd * sd);
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): arg = %s" % \
str(arg));
#%2d gaussian kernel
"""
From http://www.mathworks.com/help/matlab/ref/exp.html:
"exp(X) returns the exponential for each element of array X."
"""
#g=exp(arg)/(2*pi*sd^2); #2d gaussian kernel
g = np.exp(arg) / (2.0 * math.pi * pow(sd, 2));
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): g = %s" % \
str(g));
# normalize to suppress any gain
#if sum(g(:))~=0:
g_sum = g.sum();
#if abs(g.sum()) > 1.0e-6:
if abs(g_sum) > 1.0e-6:
#g = g / sum(g(:));
#g = g / float(g.sum());
g /= float(g_sum);
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): sd = %s" % str(sd));
common.DebugPrint("multi_scale_harris_Evangelidis(): w = %s" % str(w));
"""
#%Instead of computing derivatives in the filtered image,
we filter the image with the derivatives of the kernel.
"""
#% kernels for gaussian derivatives
#gx=-xco.*g/(sd*sd);
gx = -xco * g / float(sd * sd);
#gy=-yco.*g/(sd*sd);
gy = -yco * g / float(sd * sd);
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): gx = %s" % \
str(gx));
common.DebugPrint("multi_scale_harris_Evangelidis(): gy = %s" % \
str(gy));
"""
multi_scale_harris_Evangelidis(): arg.shape = (11, 11)
multi_scale_harris_Evangelidis(): g.shape = (11, 11)
multi_scale_harris_Evangelidis(): gx.shape = (11, 11)
multi_scale_harris_Evangelidis(): gy.shape = (11, 11)
multi_scale_harris_Evangelidis(): gi.shape = (15, 15)
"""
common.DebugPrint("multi_scale_harris_Evangelidis(): xco.shape = %s" % str(xco.shape));
common.DebugPrint("multi_scale_harris_Evangelidis(): yco.shape = %s" % str(yco.shape));
common.DebugPrint("multi_scale_harris_Evangelidis(): arg.shape = %s" % str(arg.shape));
common.DebugPrint("multi_scale_harris_Evangelidis(): g.shape = %s" % str(g.shape));
common.DebugPrint("multi_scale_harris_Evangelidis(): gx.shape = %s" % str(gx.shape));
common.DebugPrint("multi_scale_harris_Evangelidis(): gy.shape = %s" % str(gy.shape));
#%compute the derivatives
#Ix = imfilter(im, gx, 'replicate');
Ix = Matlab.imfilter(im, gx, "replicate");
#Iy = imfilter(im, gy, 'replicate');
Iy = Matlab.imfilter(im, gy, "replicate");
#% Alex: Ix and Iy have the same size as im
#if True:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): Ix = %s" % \
str(Ix));
common.DebugPrint("multi_scale_harris_Evangelidis(): Iy = %s" % \
str(Iy));
#% gaussian kernel to compute 2nd moment matrix
#if mod(floor(6*si),2) %size: six times the integration scale
if int(math.floor(6 * si)) % 2 == 1: #size: six times the integration scale
#gi = fspecial('ga',max(1,fix(6*si)), si)
gi = Matlab.fspecial("ga", max(1, Matlab.fix(6 * si)), si);
else:
#gi = fspecial('ga',max(1,fix(6*si)+1), si)
gi = Matlab.fspecial("ga", max(1, Matlab.fix(6 * si) + 1), si);
#if True:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): gi = %s" % \
str(gi));
common.DebugPrint("multi_scale_harris_Evangelidis(): gi.shape = %s" % \
str(gi.shape));
#Ix2 = imfilter(Ix.^2, gi, 'replicate');
Ix2 = Matlab.imfilter(Ix ** 2, gi, "replicate");
#Iy2 = imfilter(Iy.^2, gi, 'replicate');
Iy2 = Matlab.imfilter(Iy ** 2, gi, "replicate");
#Ixy = imfilter(Ix.*Iy, gi, 'replicate');
Ixy = Matlab.imfilter(Ix * Iy, gi, "replicate");
#% Alex: Ix2, Iy2 and Ixy have the same size as im
#if True:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): Ix2 = %s" % \
str(Ix2));
common.DebugPrint("multi_scale_harris_Evangelidis(): Iy2 = %s" % \
str(Iy2));
common.DebugPrint("multi_scale_harris_Evangelidis(): Ixy = %s" % \
str(Ixy));
common.DebugPrint("multi_scale_harris_Evangelidis(): Ix2.dtype = %s" % \
str(Ix2.dtype));
common.DebugPrint("multi_scale_harris_Evangelidis(): Iy2.dtype = %s" % \
str(Iy2.dtype));
common.DebugPrint("multi_scale_harris_Evangelidis(): Ixy.dtype = %s" % \
str(Ixy.dtype));
#%% Cornerness measure
#% Noble measure.
#%
#% M = (Ix2.*Iy2 - Ixy.^2)./(Ix2 + Iy2 + eps);
#% Harris measure.
#M = (Ix2.*Iy2 - Ixy.^2) - .06*(Ix2 + Iy2).^2;
M = (Ix2 * Iy2 - Ixy ** 2) - 0.06 * (Ix2 + Iy2) ** 2;
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): M.dtype = %s" % \
str(M.dtype));
common.DebugPrint("multi_scale_harris_Evangelidis(): M = %s" % \
str(M));
#% Alex: scn is a vector - see definition above
#% Alex: M has the same size as im
#M = scn(i)*M;
M = scn[i - 1] * M;
#thresh = 0.001*max(M(:));
thresh = 0.001 * M.max();
#% imagesc(M==abs(M));axis on;
#% colorbar
#% pause
"""
%keep points that are the maximum in a neighborhood of
radius=round(3*si/2) and are above thresh
%non-maximum supression and subpixel refinement
"""
#[r,c, rsubp, csubp] = my_nms(M, round(3*si/2), thresh);
r, c, rsubp, csubp = my_nms(M, round(3 * si / 2.0), thresh);
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): r.shape = %s" % str(r.shape));
common.DebugPrint("multi_scale_harris_Evangelidis(): r = %s" % str(r));
common.DebugPrint("multi_scale_harris_Evangelidis(): c.shape = %s" % str(c.shape));
common.DebugPrint("multi_scale_harris_Evangelidis(): c = %s" % str(c));
common.DebugPrint("multi_scale_harris_Evangelidis(): rsubp.shape = %s" % str(rsubp.shape));
common.DebugPrint("multi_scale_harris_Evangelidis(): rsubp = %s" % str(rsubp));
common.DebugPrint("multi_scale_harris_Evangelidis(): csubp.shape = %s" % str(csubp.shape));
common.DebugPrint("multi_scale_harris_Evangelidis(): csubp = %s" % str(csubp));
#% Alex: r,c, rsubp, csubp seem to always be the same size - and the
#% size I've seen is 56 * 1????
#pp=[rsubp, csubp, i*ones(size(r,1),1)];
pp = np.c_[rsubp, csubp, i * np.ones((r.shape[0], 1))];
#% Alex: here we add more rows (pp) to points below the existing rows of
#% points
#points=[points; pp];
common.DebugPrint("multi_scale_harris_Evangelidis(): points.shape = %s" % str(points.shape));
common.DebugPrint("multi_scale_harris_Evangelidis(): pp.shape = %s" % str(pp.shape));
if points.size == 0:
# Avoiding exception: "ValueError: arrays must have same number of dimensions"
points = pp;
else:
points = np.r_[points, pp];
#toc
if disp:
assert False; # not implemented the display of Harris features
"""
figure;
imshow(im,[]) #hold on
title('corners detected')
for i = range(1, size(points,1) + 1):
rectangle('Position',[points(i,2)-3*points(i,3),points(i,1)-3*points(i,3),...
2*3*points(i,3),2*3*points(i,3)],'Curvature',[1,1],'EdgeColor','w','LineWidth',2)
plot(points(i,2),points(i,1),'r+')
"""
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): points = %s" % str(points));
common.DebugPrint("multi_scale_harris_Evangelidis(): points.shape = %s" % str(points.shape));
tMSH2 = float(cv2.getTickCount());
myTime = (tMSH2 - tMSH1) / cv2.getTickFrequency();
print("multi_scale_harris_Evangelidis(): multi_scale_harris_Evangelidis() took %.6f [sec]" % myTime);
return points;
def multiscale_quad_retrieval(r_quadsTree, r_harlocs, q_harlocs, md_threshold, st_threshold, \
all_ori, all_id, all_max, all_cen, nos, scale_index, cropflag, \
sequence):
common.DebugPrint("Entered multiscale_quad_retrieval(): " \
"md_threshold = %s, st_threshold = %s." % \
(str(md_threshold), \
str(st_threshold)))
assert len(r_harlocs) != 0
assert len(q_harlocs) != 0
try:
Votes_space = np.load("Votes_space%d.npz" % scale_index)['arr_0']
HH = np.load("HH%d.npz" % scale_index)['arr_0']
return Votes_space, HH
except:
common.DebugPrintErrorTrace()
if common.MY_DEBUG_STDOUT and DBGPRINT:
common.DebugPrint("multiscale_quad_retrieval(): r_quadsTree = %s" % \
str(r_quadsTree))
common.DebugPrint("multiscale_quad_retrieval(): len(r_harlocs) = %d" %
len(r_harlocs))
common.DebugPrint("multiscale_quad_retrieval(): r_harlocs = %s" %
str(r_harlocs))
common.DebugPrint("multiscale_quad_retrieval(): q_harlocs = %s" %
str(q_harlocs))
common.DebugPrint("multiscale_quad_retrieval(): md_threshold = %s" %
str(md_threshold))
print("multiscale_quad_retrieval(): st_threshold = %s" %
str(st_threshold))
#common.DebugPrint("multiscale_quad_retrieval(): all_ori, all_id, all_max, all_cen, nos, scale_index, cropflag = %s" % str(all_ori, all_id, all_max, all_cen, nos, scale_index, cropflag));
common.DebugPrint("multiscale_quad_retrieval(): all_id = %s" %
str(all_id))
common.DebugPrint("multiscale_quad_retrieval(): all_id.shape = %s" %
(str(all_id.shape)))
#common.DebugPrint("multiscale_quad_retrieval(): all_max, all_cen, nos, scale_index, cropflag = %s" % str(all_max, all_cen, nos, scale_index, cropflag));
#common.DebugPrint("multiscale_quad_retrieval(): all_max = %s" % str(all_max));
#common.DebugPrint("multiscale_quad_retrieval(): all_cen, nos, scale_index, cropflag = %s" % str(all_cen, nos, scale_index, cropflag));
common.DebugPrint("multiscale_quad_retrieval(): sequence = %s" %
str(sequence))
print("multiscale_quad_retrieval(): cropflag = %s" % str(cropflag))
t1 = float(cv2.getTickCount())
if scale_index > nos:
assert scale_index <= nos
#error('Wrong scale index or number-of-scales');
#QD = dir([q_path "multiharlocs*.mat"])
#QD = [q_path + "multiharlocs*.mat"]
#QD = q_harlocs;
#RD = dir([r_path "multiharlocs*.mat"])
#RD = [r_path + "multiharlocs*.mat"]
#RD = r_harlocs;
#TODO: take out RD_start
#RD_start = str2num(RD(1).name(end - 9 : end - 4))
#RD_start = int(RD[0][-9 : -4])
RD_start = 0
#RD_end = str2num(RD(end).name(end - 9 : end - 4))
#RD_end = int(RD[-1][-9 : -4])
#RD_end = len(RD) - 1;
RD_end = len(r_harlocs) - 1
if False: # n_d not used anywhere
#n_d = hist(all_id, RD_start : RD_end)
#n_d = hist[all_id, RD_start : RD_end]
n_d = Matlab.hist(x=all_id, \
binCenters=np.array(range(RD_start, RD_end + 1)) )
#cross_indices = np.zeros( (len(QD), 2) );
cross_indices = np.zeros((len(q_harlocs), 2))
j = 1
#tic
#ORI = np.array([]); # ORI NOT used anywhere
"""
Inspired from
https://stackoverflow.com/questions/17559140/matlab-twice-as-fast-as-numpy
BUT doesn't help in this case:
Votes_space = np.asfortranarray(np.zeros( (len(RD), len(QD)) ));
"""
#Votes_space = np.zeros( (len(RD), len(QD)) );
Votes_space = np.zeros((len(r_harlocs), len(q_harlocs)))
# Make a distinct copy of HH from Votes_space...
#HH = Votes_space.copy().astype(np.int16); #Votes_space + 0;
#HH = np.zeros((len(RD), len(QD)), dtype=np.int8);
HH = np.zeros((len(r_harlocs), len(q_harlocs)), dtype=np.int8)
#!!!!TODO use MAYBE even np.bool - OR take it out
#common.DebugPrint("multiscale_quad_retrieval(): Votes_space = %s,\n HH = %s" % (str(Votes_space), str(HH)))
tolers = 0.1 - float(scale_index) / 100.0
# it helps to make more strict the threshold as the scale goes up
# tolers = 0.15 - float(scale_index) / 100.0;
MAXDIS = 3 + scale_index
MAXORI = 0.25
"""
!!!!TODO TODO: I am using multiprocessing.Poll and return votes;
the dispatcher assembles the results,
but the results are NOT the same with the serial case - although they
look pretty decent, but they seem to be suboptimal - dp_Alex returns
suboptimal cost path for USE_MULTITHREADING == True instead of
False.
(Note: running under the same preconditions
multiscale_quad_retrieval I got the same results in dp_Alex().
"""
if False: #config.USE_MULTITHREADING == True:
global g
g.r_quadsTree = r_quadsTree
g.r_harlocs = r_harlocs
g.q_harlocs = q_harlocs
g.md_threshold = md_threshold
g.st_threshold = st_threshold
g.all_ori = all_ori
g.all_id = all_id
g.all_max = all_max
g.all_cen = all_cen
g.nos = nos
g.scale_index = scale_index
g.cropflag = cropflag
g.sequence = sequence
g.RD_start = RD_start
g.RD_end = RD_end
g.MAXDIS = MAXDIS
g.MAXORI = MAXORI
g.tolers = tolers
"""
Start worker processes to use on multi-core processor (able to run
in parallel - no GIL issue if each core has it's own VM)
"""
pool = multiprocessing.Pool(processes=config.numProcesses)
print("multiscale_quad_retrieval(): Spawned a pool of %d workers" % \
config.numProcesses)
listParams = range(0, len(q_harlocs))
#!!!!TODO: use counterStep, config.initFrame[indexVideo]
#res = pool.map(IterationStandaloneMQR, listParams);
# See https://docs.python.org/2/library/multiprocessing.html#module-multiprocessing.pool
res = pool.map(func=IterationStandaloneMQR, iterable=listParams, \
chunksize=1)
print("Pool.map returns %s" % str(res))
#x0.size + 1
"""
From https://medium.com/building-things-on-the-internet/40e9b2b36148
close the pool and wait for the work to finish
"""
pool.close()
pool.join()
# Doing the "reduce" phase after the workers have finished :)
assert len(res) == len(q_harlocs)
for queryFrame, resE in enumerate(res):
resEIndex = resE[0]
resE = resE[1]
assert resEIndex == queryFrame
# Gives: "ValueError: output operand requires a reduction, but reduction is not enabled"
#Votes_space[:, queryFrame - 1] = votes;
Votes_space[:, queryFrame] = resE
for queryFrame in range(len(q_harlocs)):
if cropflag == 0:
HH[:, queryFrame] = 1
else:
"""
HH[:, queryFrame] = spatial_consistency.spatial_consistency(space_xy, \
qcen, len(RD), st_threshold, cropflag);
"""
HH[:, queryFrame] = spatial_consistency.spatial_consistency(space_xy, \
qcen, len(r_harlocs), st_threshold, cropflag)
try:
np.savez_compressed("Votes_space%d" % scale_index, Votes_space)
np.savez_compressed("HH%d" % scale_index, HH)
except:
common.DebugPrintErrorTrace()
return Votes_space, HH
"""
We substitute q - 1 with q, since we want
to number arrays from 0 (not from 1 like in Matlab).
"""
#for q=1:length(QD)
#for q in range(1, len(QD) + 1):
#for queryFrame in range(len(QD)):
for queryFrame in range(len(q_harlocs)):
common.DebugPrint(
"multiscale_quad_retrieval(): Starting iteration queryFrame = %d" %
queryFrame)
# tic
"""
str1=['load ' q_path QD(q).name]
eval(str1)
"""
"""
We make pp reference the desired multiharloc list for the query video
frame queryFrame
"""
pp = q_harlocs[queryFrame]
#pp = np.array(pp);
#common.DebugPrint("multiscale_quad_retrieval(): pp = %s" % str(pp));
#[qout,qcen,qmaxdis,qori]=findquads(pp(pp(:,3)==scale_index,1:2),md_threshold,0);
points = pp[pp[:, 2] == scale_index, 0:2]
qout, qcen, qmaxdis, qori = findquads.findquads(
points, md_threshold, 0)
if common.MY_DEBUG_STDOUT and DBGPRINT:
print("multiscale_quad_retrieval(): queryFrame = %d, " \
"qout.shape (number of quads for query frame queryFrame) = %s" % \
(queryFrame, str(qout.shape)))
# disp([num2str(q) ' of ' num2str(length(QD)) ' -> ' num2str(size(qout,1)) ' quads'])
#space_xy=zeros(size(qcen,1),2*length(RD))+nan;
#space_xy = np.zeros( (qcen.shape[0], 2 * len(RD)) ) + np.nan;
space_xy = np.zeros((qcen.shape[0], 2 * len(r_harlocs))) + np.nan
# votes=zeros(length(RD),1)
#votes=zeros(length(RD),length(tolers));
#votes = np.zeros( (len(RD), 1) );
votes = np.zeros((len(r_harlocs), 1))
#nep = np.array([]);
#m_points = np.array([]);
assert isinstance(tolers, float)
if common.MY_DEBUG_STDOUT:
common.DebugPrint(
"multiscale_quad_retrieval(): quads of query frame %d are: " %
queryFrame)
common.DebugPrint(" qout = %s" % str(qout))
"""
Alex: for each quad (4 floats) of the query frame from Harris feature of scale scale_index
Note: all_id stores the reference frame id for each quad descriptor.
"""
"""
We substitute queryFrameQuad - 1 with queryFrameQuad, since we want
to number arrays from 0 (not from 1 like in Matlab).
"""
#for queryFrameQuad in range(1, qout.shape[0] + 1):
for queryFrameQuad in range(qout.shape[0]):
common.DebugPrint(
"multiscale_quad_retrieval(): Starting iteration queryFrameQuad = %d"
% queryFrameQuad)
"""
Matlab's polymorphism is really bugging here: although it's
normally a float, tolers is considered to be a size 1 vector...
so len(tolers) == 1
"""
#for tol_i in range(1, len(tolers) + 1):
# tol = tolers[tol_i - 1]
"""
We substitute tol_i - 1 with tol, since we want
to number arrays from 0 (not from 1 like in Matlab).
"""
#for tol_i in range(1, 1 + 1):
for tol_i in range(1):
tol = tolers
"""
# TODO: done below - take out this dbg print
if DBGPRINT:
common.DebugPrint("multiscale_quad_retrieval(): " \
"qout[queryFrameQuad, :] = %s" % \
str(qout[queryFrameQuad, :]))
"""
#% default for first PAMI with tol= 0.1 approximately
# NOTE: SciPy's KDTree finds a few more results, in some cases,
# than the Matlab code from Evangelidis.
#idx, di = kdtree_ball_query(tree, qout(i, :), tol)
#idx, distKD = kdtree_ball_query(tree, qout[i - 1, :], tol)
#idx, di = tree.query(x=xQuery, k=4)
#resPoints = [data[i] for i in resBallIndices]
# tol is a scalar representing the radius of the ball
if config.KDTREE_IMPLEMENTATION == 0:
idx = r_quadsTree.query_ball_point(qout[queryFrameQuad, :],
tol)
elif config.KDTREE_IMPLEMENTATION == 1:
#pt = qout[queryFrameQuad - 1, :].astype(np.float32);
pt = qout[queryFrameQuad, :]
pt = np.array([[pt[0], pt[1], pt[2], pt[3]]],
dtype=np.float32)
retval, idx, dists = r_quadsTree.radiusSearch( \
query=pt, \
radius=(tol**2), \
maxResults=NUM_MAX_ELEMS, \
params=search_params)
if common.MY_DEBUG_STDOUT and DBGPRINT:
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"retval (number NNs) = %s" % str(retval));
"""
common.DebugPrint( \
"multiscale_quad_retrieval(): radiusSearch's retval " \
"(at queryFrame=%d, queryFrameQuad=%d) is %d" % (queryFrame, queryFrameQuad, retval))
idx = idx[0]
dists = dists[0]
"""
Note: retval is the number of neighbors returned from the radiusSearch().
But the idx and the dists can have more elements than the returned retval.
"""
idx = idx[:retval]
dists = dists[:retval]
if common.MY_DEBUG_STDOUT and DBGPRINT:
print("multiscale_quad_retrieval(): " \
"qout[queryFrameQuad, :] = %s" % str(qout[queryFrameQuad, :]))
print("multiscale_quad_retrieval(): " \
"idx = %s" % str(idx))
print("multiscale_quad_retrieval(): " \
"dists = %s" % str(dists))
print("multiscale_quad_retrieval(): " \
"tol = %s" % str(tol))
if config.KDTREE_IMPLEMENTATION == 0:
print("multiscale_quad_retrieval(): " \
"r_quadsTree.data[idx] = %s" % \
str(r_quadsTree.data[idx]))
# We print the distances to the points returned in idx
if common.MY_DEBUG_STDOUT and DBGPRINT: # This is just for debugging purposes
a = qout[queryFrameQuad, :]
if config.KDTREE_IMPLEMENTATION == 0:
for myI, index in enumerate(idx):
b = r_quadsTree.data[index]
"""
if False:
common.DebugPrint("multiscale_quad_retrieval(): distance to " \
"%d point (%s) inside ball = %.4f" % \
(myI, str(b), npla.norm(a - b)));
"""
else:
pass
idx = np.array(idx)
#if False:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): " \
"all_max.shape = %s" % str(all_max.shape))
common.DebugPrint("multiscale_quad_retrieval(): " \
"qmaxdis.shape = %s" % str(qmaxdis.shape))
common.DebugPrint("multiscale_quad_retrieval(): " \
"qmaxdis = %s" % str(qmaxdis))
common.DebugPrint("multiscale_quad_retrieval(): " \
"qori.shape = %s" % str(qori.shape))
common.DebugPrint("multiscale_quad_retrieval(): " \
"qori = %s" % str(qori))
#dis_idx=abs(qmaxdis(i)-all_max(idx))<MAXDIS;
if len(idx) == 0:
# NOT A GOOD IDEA: continue;
#idx = np.array([]);
dis_idx = np.array([])
ori_idx = np.array([])
else:
if common.MY_DEBUG_STDOUT and DBGPRINT:
print("multiscale_quad_retrieval(): " \
"queryFrameQuad = %s" % str(queryFrameQuad))
print("multiscale_quad_retrieval(): " \
"all_max[idx] = %s" % str(all_max[idx]))
print("multiscale_quad_retrieval(): " \
"qmaxdis[queryFrameQuad] = %s" % str(qmaxdis[queryFrameQuad]))
if USE_GPS_COORDINATES:
# We look only at a part of the reference video
"""
Since in some cases the video temporal alignment is
difficult to do due to similar portions in the
trajectory (see the drone videos, clip 3_some_lake)
we "guide" the temporal alignment by restricting
the reference frame search space - this is useful
when we have the geolocation (GPS) coordinate for
each frame.
"""
if common.MY_DEBUG_STDOUT and DBGPRINT:
print("multiscale_quad_retrieval(): " \
"all_id = %s" % str(all_id))
if True:
#assert (all_id.ndim == 2) and (all_id.shape[1] == 1);
if all_id.ndim == 2:
#!!!!TODO TODO: put this at the beginning of the function
assert all_id.shape[1] == 1
"""
We flatten the array all_id
Note: We don't use order="F" since it's
basically 1-D array
"""
all_id = np.ravel(all_id)
#!!!!TODO: put start and end frame in config - or compute it from geolocation
sub_idx = np.logical_and( (all_id[idx] >= 2030 - 928), \
(all_id[idx] <= 2400 - 928) )
idx = idx[sub_idx]
if common.MY_DEBUG_STDOUT and DBGPRINT:
print("multiscale_quad_retrieval(): " \
"all_id = %s" % str(all_id))
print("multiscale_quad_retrieval(): " \
"sub_idx = %s" % str(sub_idx))
print("multiscale_quad_retrieval(): " \
"idx = %s" % str(idx))
if FILTER:
dis_idx = np.abs(qmaxdis[queryFrameQuad] -
all_max[idx]) < MAXDIS
#if False:
if common.MY_DEBUG_STDOUT:
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"idx = %s" % str(idx));
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"dis_idx = %s" % str(dis_idx))
#idx=idx(dis_idx)
idx = idx[dis_idx]
#if False:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): " \
"idx (after idx = idx[dis_idx]) = %s" % str(idx))
if FILTER:
#ori_idx=abs(qori(i)-all_ori(idx))<MAXORI;
ori_idx = np.abs(qori[queryFrameQuad] -
all_ori[idx]) < MAXORI
#if False:
if common.MY_DEBUG_STDOUT:
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"all_ori = %s" % str(all_ori));
common.DebugPrint("multiscale_quad_retrieval(): " \
"qori[queryFrameQuad] = %s" % str(qori[queryFrameQuad]));
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"ori_idx = %s" % str(ori_idx))
#idx=idx(ori_idx);
idx = idx[ori_idx]
# IMPORTANT ###################################################
# IMPORTANT ###################################################
# IMPORTANT ###################################################
#% spatio-temporal consistency
# IMPORTANT ###################################################
# IMPORTANT ###################################################
# IMPORTANT ###################################################
#if numel(idx) > 0:
if idx.size > 0:
if cropflag == 0:
if FILTER:
"""
Alex: this is a simple procedure of eliminating False
Positive (FP) matches, as presented in Section 4.2 of
TPAMI 2013 paper.
Basically it filters out quad matches that have
centroids st_threshold away from the query quad.
Note: all_cen are the controids of all reference
quads.
"""
dy = qcen[queryFrameQuad, 0] - all_cen[idx, 0]
dx = qcen[queryFrameQuad, 1] - all_cen[idx, 1]
#D=dy.^2+dx.^2;
D = dy**2 + dx**2
co_idx = D < pow(st_threshold, 2)
idx = idx[co_idx]
else:
"""
We substitute iii - 1 with iii, since we want
to number arrays from 0 (not from 1 like in Matlab).
"""
#for iii in range(1, len(idx) + 1):
for iii in range(len(idx)):
#space_xy(i,(all_id(idx(iii))-RD_start)*2+1:(all_id(idx(iii))-RD_start)*2+2) = all_cen(idx(iii),:)
space_xy[queryFrameQuad, \
(all_id[idx[iii]] - RD_start) * 2: (all_id[idx[iii] - 1] - RD_start) * 2 + 1] = \
all_cen[idx[iii], :]
#hh=hist(all_id(idx),RD_start:RD_end);
# It has to be an np.array because we multiply it with a scalar
histoRange = np.array(range(RD_start, RD_end + 1))
hh = Matlab.hist(x=all_id[idx], binCenters=histoRange)
#if False:
#if True:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): " \
"hh = %s" % (str(hh)))
common.DebugPrint("multiscale_quad_retrieval(): " \
"hh.shape = %s" % (str(hh.shape)))
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"all_id = %s" % (str(all_id)));
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"all_id.shape = %s" % (str(all_id.shape)))
common.DebugPrint("multiscale_quad_retrieval(): " \
"idx = %s" % (str(idx)))
common.DebugPrint("multiscale_quad_retrieval(): " \
"idx.shape = %s" % (str(idx.shape)))
# % nz can be computed more optimally
#nz=find(hh~=0); # nz can be computed more optimally
# np.nonzero() always returns a tuple, even if it contains 1 element since hh has only 1 dimension
nz = np.nonzero(hh != 0)[0]
#if False:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): " \
"nz = %s" % (str(nz)))
common.DebugPrint("multiscale_quad_retrieval(): " \
"nz.shape = %s" % (str(nz.shape)))
#if numel(nz) > 0
if nz.size > 0:
#%%----text-retrieval-like
#votes(nz, tol_i) = votes(nz, tol_i) + log10(length(RD) / (length(nz)))^2 #PREVIOUSLY
#myVal = pow(math.log10(float(len(RD)) / len(nz)), 2);
myVal = pow(
math.log10(float(len(r_harlocs)) / len(nz)), 2)
"""
try:
myVal = pow(math.log10(float(len(r_harlocs)) / len(nz)), 2);
except:
print("Error: len=%d len(nz)=%d nz.size=%d" % \
(len(r_harlocs), len(nz), nz.size));
common.DebugPrintErrorTrace();
"""
#if False:
if common.MY_DEBUG_STDOUT:
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"len(RD) = %d" % len(RD));
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"len(r_harlocs) = %d" % len(r_harlocs))
common.DebugPrint("multiscale_quad_retrieval(): " \
"len(nz) = %d" % len(nz))
common.DebugPrint("multiscale_quad_retrieval(): " \
"myVal = %.5f" % myVal)
# PREVIOUSLY
votes[nz, tol_i] = votes[nz, tol_i] + myVal
# votes(nz)=votes(nz)+log10(length(RD)/(length(nz)));
# votes(nz)=votes(nz)+1;
if common.MY_DEBUG_STDOUT and DBGPRINT:
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"Votes_space.shape = %s" % (str(Votes_space.shape)));
common.DebugPrint("multiscale_quad_retrieval(): " \
"votes.shape = %s" % (str(votes.shape)));
"""
print("multiscale_quad_retrieval(): " \
"votes.shape = %s" % (str(votes.shape)))
if (np.abs(votes) < 1.0e-10).all():
print( \
"multiscale_quad_retrieval(): votes = 0 (all zeros)")
else:
print("multiscale_quad_retrieval(): " \
"votes = %s" % (str(votes)))
#Votes_space(:,q)=votes;
# Gives: "ValueError: output operand requires a reduction, but reduction is not enabled"
#Votes_space[:, queryFrame - 1] = votes;
# Note: since votes is basically a 1-D vector, we don't use the Fortran order
Votes_space[:, queryFrame] = np.ravel(votes)
# order="F");
if cropflag == 0:
HH[:, queryFrame] = 1
else:
"""
HH[:, queryFrame] = spatial_consistency.spatial_consistency(space_xy, \
qcen, len(RD), st_threshold, cropflag);
"""
HH[:, queryFrame] = spatial_consistency.spatial_consistency(space_xy, \
qcen, len(r_harlocs), st_threshold, cropflag)
if common.MY_DEBUG_STDOUT and DBGPRINT:
print("multiscale_quad_retrieval(scale_index=%d): " \
"Votes_space =\n%s" % (scale_index, str(Votes_space)))
try:
np.savez_compressed("Votes_space%d" % scale_index, Votes_space)
np.savez_compressed("HH%d" % scale_index, HH)
except:
common.DebugPrintErrorTrace()
t2 = float(cv2.getTickCount())
myTime = (t2 - t1) / cv2.getTickFrequency()
print("multiscale_quad_retrieval() took %.6f [sec]" % myTime)
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"%d corresponding frames retrieved in %.6f secs" % \
(len(q_harlocs), myTime));
"""
return Votes_space, HH
def dp3(Vspace, numFramesR, numFramesQ, BOV_flag):
#% Dynamic programming for a maximum-vote path in vote-space
#% 2010, Georgios Evangelidis <*****@*****.**>
print ("Entered dp3(): Running dynamic programming...")
if common.MY_DEBUG_STDOUT:
common.DebugPrint("dp3(): Vspace = %s" % str(Vspace))
# tic
# QD=dir([q_path 'multiharlocs*.mat']);
# RD=dir([r_path 'multiharlocs*.mat']);
# cross=zeros(length(QD),2);
# crossref = np.zeros( (len(q_path), 2) );
crossref = np.zeros((numFramesQ, 2))
# sigma_0=1.2;
sigma_0 = 1.2
# [r,c,d] = size(Vspace);
r, c, d = Vspace.shape
# n=[1:d]; %scale levels
n = np.array(range(1, d + 1))
#%scale levels
# sigma_I=sqrt(1.8).^n*sigma_0;
sigma_I = math.sqrt(1.8) ** n * sigma_0
w = sigma_I
#%w=w*0+1;
# w=w/sum(w);
w = w / float(w.sum())
if BOV_flag == 1:
# w=fliplr(w);
w = w[:, ::-1]
#% Initialization
# D = zeros(r+1, c+1);
D = np.zeros((r + 1, c + 1))
# D(1,:) = NaN;
D[0, :] = np.nan
# D(:,1) = NaN;
D[:, 0] = np.nan
# D(1,1) = 0;
D[0, 0] = 0
# for i=1:d
"""
We substitute i - 1 with i since arrays start with 0 in Python,
not with 1 like in Matlab.
"""
# for i in range(1, d + 1):
for i in range(d):
# V_temp=Vspace(:,:,i);
V_temp = Vspace[:, :, i]
# V_temp(isnan(V_temp))=0;
V_temp[np.isnan(V_temp)] = 0
# !!!!TODO: check OK
# Vspace(:,:,i)=V_temp;
Vspace[:, :, i] = V_temp
# VV=zeros(r,c);
VV = np.zeros((r, c))
# for j=1:d
"""
We substitute j - 1 with j since arrays start with 0 in Python,
not with 1 like in Matlab.
"""
# for j in range(1, d + 1):
for j in range(d):
# VV=VV+Vspace(:,:,j);
VV = VV + Vspace[:, :, j]
# D(2:end,2:end)=VV;
D[1:, 1:] = VV
# NEW_DP3_ALEX = True;
NEW_DP3_ALEX = False
if NEW_DP3_ALEX:
# Alex: added cost
cost = np.zeros((r + 1, c + 1))
#% for traceback
# Tback = zeros(r+1,c+1);
Tback = np.zeros((r + 1, c + 1))
# if True:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("dp3(): printing locally optimum solutions:")
# Alex: trying out to find a better solution than dp3() !!!!TODO : more
# This solution is basically the one returned by causal() IF we do NOT apply Matlab.filter2() on Vspace
for j in range(1, c + 1):
maxCol = 0.0
maxPos = -1
for i in range(1, r + 1):
assert D[i, j] >= 0.0
if maxCol < D[i, j]:
maxCol = D[i, j]
maxPos = i
# So for query frame j we have a candidate matching ref frame i
common.DebugPrint("dp3(): for query frame %d - candidate frame %d" % (j - 1, maxPos))
common.DebugPrint("dp3(): for query frame %d we found matching ref frame %d" % (j - 1, maxPos))
# !!!!TODO: make i =0.., j=0.. and substitute i-1 with i, i-2 with i-1, etc
# for i = 1:r;
for i in range(1, r + 1):
# for j = 1:c
for j in range(1, c + 1):
# if (i>1) && (j>1)
if (i > 1) and (j > 1):
# dd1 = w(1)*Vspace(i-1,max(1,j-2),1);
dd1 = w[0] * Vspace[i - 2, max(0, j - 3), 0]
# dd2 = w(1)*Vspace(max(1,i-2),j-1,1);
dd2 = w[0] * Vspace[max(0, i - 3), j - 2, 0]
# dd3 = w(1)*Vspace(i-1,j-1,1);
dd3 = w[0] * Vspace[i - 2, j - 2, 0]
# dd4 = w(1)*Vspace(i-1,j,1);
dd4 = w[0] * Vspace[i - 2, j - 1, 0]
# dd5 = w(1)*Vspace(i,j-1,1);
dd5 = w[0] * Vspace[i - 1, j - 2, 0]
if d > 1:
# for sc = 2:d
for sc in range(2, d + 1):
# dd1 = max(dd1, w(sc)*Vspace(i-1,max(1,j-2),sc));
dd1 = max(dd1, w[sc - 1] * Vspace[i - 2, max(0, j - 3), sc - 1])
# dd2 = max(dd2, w(sc)*Vspace(max(1,i-2),j-1,sc));
dd2 = max(dd2, w[sc - 1] * Vspace[max(0, i - 3), j - 2, sc - 1])
# dd3 = max(dd3, w(sc)*Vspace(i-1,j-1,sc));
dd3 = max(dd3, w[sc - 1] * Vspace[i - 2, j - 2, sc - 1])
# dd4 = max(dd4, w(sc)*Vspace(i-1,j,sc));
dd4 = max(dd4, w[sc - 1] * Vspace[i - 2, j - 1, sc - 1])
# dd5 = max(dd5, w(sc)*Vspace(i,j-1,sc));
dd5 = max(dd5, w[sc - 1] * Vspace[i - 1, j - 2, sc - 1])
# D(i,j-1)=D(i,j-1)+dd1;
D[i - 1, j - 2] += dd1
# D(i-1,j)=D(i-1,j)+dd2;
D[i - 2, j - 1] += dd2
# D(i,j)=D(i,j)+ dd3;
D[i - 1, j - 1] += dd3
# D(i,j+1)=D(i,j+1)+ dd4;
D[i - 1, j] += dd4
# D(i+1,j)=D(i+1,j)+ dd5;
D[i, j - 1] += dd5
#% % instead of above loop, use the following five lines when scales=6
#% D(i,j-1)=D(i,j-1)+max([w(1)*Vspace(i-1,max(1,j-2),1),w(2)*Vspace(i-1,max(1,j-2),2),w(3)*Vspace(i-1,max(1,j-2),3),w(4)*Vspace(i-1,max(1,j-2),4),w(5)*Vspace(i-1,max(1,j-2),5),w(6)*Vspace(i-1,max(1,j-2),6)]);
#% D(i-1,j)=D(i-1,j)+max([w(1)*Vspace(max(1,i-2),j-1,1),w(2)*Vspace(max(1,i-2),j-1,2),w(3)*Vspace(max(1,i-2),j-1,3),w(4)*Vspace(max(1,i-2),j-1,4),w(5)*Vspace(max(1,i-2),j-1,5),w(6)*Vspace(max(1,i-2),j-1,6)]);
#% D(i,j)=D(i,j)+max([w(1)*Vspace(i-1,j-1,1),w(2)*Vspace(i-1,j-1,2),w(3)*Vspace(i-1,j-1,3),w(4)*Vspace(i-1,j-1,4),w(5)*Vspace(i-1,j-1,5),w(6)*Vspace(i-1,j-1,6)]);
#% D(i,j+1)=D(i,j+1)+max([w(1)*Vspace(i-1,j,1),w(2)*Vspace(i-1,j,2),w(3)*Vspace(i-1,j,3),w(4)*Vspace(i-1,j,4),w(5)*Vspace(i-1,j,5),w(6)*Vspace(i-1,j,6)]);
#% D(i+1,j)=D(i+1,j)+max([w(1)*Vspace(i,j-1,1),w(2)*Vspace(i,j-1,2),w(3)*Vspace(i,j-1,3),w(4)*Vspace(i,j-1,4),w(5)*Vspace(i,j-1,5),w(6)*Vspace(i,j-1,6)]);
#% [dmax, tb] = max([D(i, j), D(i-1, j), D(i, j-1), D(i,j+1),D(i+1,j)]);
# [dmax, tb] = max([D(i, j)+1/sqrt(2), D(i-1, j)+1/sqrt(5), D(i, j-1)+1/sqrt(5), D(i,j+1)+1, D(i+1,j)+1]);
dmax, tb = Matlab.max(
np.array(
[
D[i - 1, j - 1] + 1.0 / math.sqrt(2.0),
D[i - 2, j - 1] + 1.0 / math.sqrt(5.0),
D[i - 1, j - 2] + 1.0 / math.sqrt(5.0),
D[i - 1, j] + 1,
D[i, j - 1] + 1,
]
)
)
else:
# [dmax, tb] = max([D(i,j), D(i,j+1), D(i+1,j)]);
dmax, tb = Matlab.max(np.array([D[i - 1, j - 1], D[i - 1, j], D[i, j - 1]]))
if common.MY_DEBUG_STDOUT:
common.DebugPrint("dp3(): dmax = %s" % str(dmax))
common.DebugPrint("dp3(): tb = %s" % str(tb))
# D(i+1,j+1) = D(i+1,j+1)+dmax;
if NEW_DP3_ALEX:
cost[i, j] = 0
#!!!!TODO: think more
else:
D[i, j] += dmax
#!!!!TODO: for me it's weird he adds dmax here...
# Tback(i+1,j+1) = tb;
Tback[i, j] = tb
if common.MY_DEBUG_STDOUT:
common.DebugPrint("dp3(): D.shape = %s" % str(D.shape))
common.DebugPrint("dp3(): D = %s" % str(D))
common.DebugPrint("dp3(): Tback.shape = %s" % str(Tback.shape))
common.DebugPrint("dp3(): Tback = %s" % str(Tback))
#% Traceback
i = r + 1
j = c + 1
y = i - 1
x = j - 1
# while i > 2 & j > 2
while (i > 2) and (j > 2):
# tb = Tback(i,j);
tb = Tback[i - 1, j - 1] + 1
# In Matlab, max returns indices from 1..
if tb == 1:
i -= 1
j -= 1
elif tb == 2:
i -= 2
j -= 1
elif tb == 3:
i -= 1
j -= 2
elif tb == 4:
i -= 1
j = j
elif tb == 5:
j -= 1
i = i
else:
# error;
assert False
# y = [i,y];
# NOT GOOD: y = np.c_[i, y];
y = np.hstack([i - 1, y])
# x = [j,x];
# NOT GOOD: x = np.c_[j, x];
x = np.hstack([j - 1, x])
if common.MY_DEBUG_STDOUT:
common.DebugPrint("dp3(): before D.shape = %s" % str(D.shape))
common.DebugPrint("dp3(): before D = %s" % str(D))
#% Strip off the edges of the D matrix before returning
# D = D(2:(r+1),2:(c+1));
D = D[1 : (r + 1), 1 : (c + 1)]
if common.MY_DEBUG_STDOUT:
common.DebugPrint("dp3(): D.shape = %s" % str(D.shape))
common.DebugPrint("dp3(): D = %s" % str(D))
# RD_start=str2num(RD(1).name(end-9:end-4));
RD_start = 1
if common.MY_DEBUG_STDOUT:
common.DebugPrint("dp3(): Vspace.shape = %s" % str(Vspace.shape))
#!!!!TODO: understand well what is x,y and why computes p
# for i=1:size(Vspace,2):
for i in range(0, Vspace.shape[1]):
# cross(i,1)=str2num(QD(i).name(end-9:end-4));
crossref[i, 0] = i
# i; #!!!!TODO: think if OK
# p=find(x==i);
p = np.nonzero(x == i)
p = p[0]
if common.MY_DEBUG_STDOUT:
common.DebugPrint("dp3(): x.shape = %s" % str(x.shape))
common.DebugPrint("dp3(): x = %s" % str(x))
common.DebugPrint("dp3(): y.shape = %s" % str(y.shape))
common.DebugPrint("dp3(): y = %s" % str(y))
common.DebugPrint("dp3(): i = %s" % str(i))
common.DebugPrint("dp3(): p = %s" % str(p))
# if isempty(p)
if p.size == 0:
#% Alex: Vali Codreanu said to change from temp=0; to temp=3;
# temp=3;
temp = 0
if common.MY_DEBUG_STDOUT:
common.DebugPrint("dp3(): temp = %s" % str(temp))
crossref[i, 1] = 0 + RD_start - 1
else:
# temp=y(p);
temp = y[p]
if common.MY_DEBUG_STDOUT:
common.DebugPrint("dp3(): temp = %s" % str(temp))
# cross(i,2)=temp(end)+RD_start-1;
if temp.size == 1:
# If temp has only 1 element:
crossref[i, 1] = temp + RD_start - 1
else:
crossref[i, 1] = temp[-1] + RD_start - 1
# assert temp.size == 1;
# common.DebugPrint("dp3(): temp = %s" % str(temp));
"""
#cross(i,2)=temp(end)+RD_start-1;
if True:
crossref[i, 1] = temp[-1] + RD_start - 1;
else:
# BAD IDEA!!!!TODO
crossref[i, 1] = temp + RD_start - 1;
"""
# printf("dp3(): Done in blabla secs\n");
common.DebugPrint("dp3(): crossref = %s" % str(crossref))
ComputeCost(crossref, VV, "crossref_dp3.txt")
# return [y,x,D,Tback,cross]
return y, x, D, Tback, crossref
def MyImageReadMSH(index):
global g
"""
common.DebugPrint("MyImageReadMSH(): initFrame = %d)" % \
(config.initFrame[g.indexVideo]));
index += config.initFrame[g.indexVideo];
"""
"""
common.DebugPrint("Entered MyImageReadMSH(capture=%s, index=%s)" % \
(str(capture), str(index)));
"""
common.DebugPrint("Entered MyImageReadMSH(index=%s)" % \
(str(index)))
"""
We must reopen the capture device in each different process, otherwise the
program blocks at the first operation in the "global" capture device.
"""
if g.harlocsFolder.endswith(config.HARRIS_QUERY_FOLDER_NAME):
if g.captureQ == None:
capture = cv2.VideoCapture(sys.argv[1])
g.captureQ = capture
common.DebugPrint("MyImageReadMSH(): new capture=%s" % \
(str(capture)))
else:
capture = g.captureQ
elif g.harlocsFolder.endswith(config.HARRIS_REFERENCE_FOLDER_NAME):
if g.captureR == None:
capture = cv2.VideoCapture(sys.argv[2])
g.captureR = capture
common.DebugPrint("MyImageReadMSH(): new capture=%s" % \
(str(capture)))
else:
capture = g.captureR
else:
assert False
assert (g.indexVideo == 0) or (g.indexVideo == 1)
if config.OCV_OLD_PY_BINDINGS:
capture.set(cv2.cv.CV_CAP_PROP_POS_FRAMES, \
index)
else:
"""
From http://docs.opencv.org/modules/highgui/doc/reading_and_writing_images_and_video.html#videocapture-get:
<<CV_CAP_PROP_POS_FRAMES 0-based index of the frame to be
decoded/captured next.>>
"""
capture.set(cv2.CAP_PROP_POS_FRAMES, \
index)
common.DebugPrint("MyImageReadMSH(): after capture.set()")
# This is only for (paranoid) testing purposes:
if config.OCV_OLD_PY_BINDINGS:
indexCrt = capture.get(cv2.cv.CV_CAP_PROP_POS_FRAMES)
else:
"""
From http://docs.opencv.org/modules/highgui/doc/reading_and_writing_images_and_video.html#videocapture-get:
<<CV_CAP_PROP_POS_FRAMES 0-based index of the frame to be
decoded/captured next.>>
"""
indexCrt = capture.get(cv2.CAP_PROP_POS_FRAMES)
#assert int(indexCrt) == index;
#!!!!TODO: think if OK
if int(indexCrt) != index:
common.DebugPrint( \
"MyImageReadMSH(): indexCrt != index --> returning black frame")
ret = False
else:
#common.DebugPrint("Alex: frameR = %d" % frameR);
#if myIndex > numFramesR:
# break;
#ret, img = r_path.read();
ret, img = capture.read()
#if ret == False:
# break;
#!!!!TODO: think if well
#assert ret == True;
if ret == False:
common.DebugPrint(
"MyImageReadMSH(index=%d): ret == False --> returning None" %
index)
img = None
else:
common.DebugPrint("MyImageReadMSH(): img.shape = %s" % str(img.shape))
common.DebugPrint("MyImageReadMSH(): img.dtype = %s" % str(img.dtype))
#!!!!TODO: I suggest to do the gray conversion at reading, not in multi_scale_harris.py
if False:
# In the Matlab code he reads gray/8bpp JPEGs
imgGray = common.ConvertImgToGrayscale(img)
if config.VIDEO_FRAME_RESIZE_SCALING_FACTOR != 1:
# We resize the image
img = Matlab.imresize(img, \
scale=config.VIDEO_FRAME_RESIZE_SCALING_FACTOR)
common.DebugPrint("Exiting MyImageReadMSH()")
if False:
return imgGray
return img
def dp3(Vspace, numFramesR, numFramesQ, BOV_flag):
#% Dynamic programming for a maximum-vote path in vote-space
#% 2010, Georgios Evangelidis <*****@*****.**>
print("Entered dp3(): Running dynamic programming...")
if common.MY_DEBUG_STDOUT:
common.DebugPrint("dp3(): Vspace = %s" % str(Vspace))
#tic
#QD=dir([q_path 'multiharlocs*.mat']);
#RD=dir([r_path 'multiharlocs*.mat']);
#cross=zeros(length(QD),2);
#crossref = np.zeros( (len(q_path), 2) );
crossref = np.zeros((numFramesQ, 2))
#sigma_0=1.2;
sigma_0 = 1.2
#[r,c,d] = size(Vspace);
r, c, d = Vspace.shape
#n=[1:d]; %scale levels
n = np.array(range(1, d + 1))
#%scale levels
#sigma_I=sqrt(1.8).^n*sigma_0;
sigma_I = math.sqrt(1.8)**n * sigma_0
w = sigma_I
#%w=w*0+1;
#w=w/sum(w);
w = w / float(w.sum())
if BOV_flag == 1:
#w=fliplr(w);
w = w[:, ::-1]
#% Initialization
#D = zeros(r+1, c+1);
D = np.zeros((r + 1, c + 1))
#D(1,:) = NaN;
D[0, :] = np.nan
#D(:,1) = NaN;
D[:, 0] = np.nan
#D(1,1) = 0;
D[0, 0] = 0
#for i=1:d
"""
We substitute i - 1 with i since arrays start with 0 in Python,
not with 1 like in Matlab.
"""
#for i in range(1, d + 1):
for i in range(d):
#V_temp=Vspace(:,:,i);
V_temp = Vspace[:, :, i]
#V_temp(isnan(V_temp))=0;
V_temp[np.isnan(V_temp)] = 0
# !!!!TODO: check OK
#Vspace(:,:,i)=V_temp;
Vspace[:, :, i] = V_temp
#VV=zeros(r,c);
VV = np.zeros((r, c))
#for j=1:d
"""
We substitute j - 1 with j since arrays start with 0 in Python,
not with 1 like in Matlab.
"""
#for j in range(1, d + 1):
for j in range(d):
#VV=VV+Vspace(:,:,j);
VV = VV + Vspace[:, :, j]
#D(2:end,2:end)=VV;
D[1:, 1:] = VV
#NEW_DP3_ALEX = True;
NEW_DP3_ALEX = False
if NEW_DP3_ALEX:
# Alex: added cost
cost = np.zeros((r + 1, c + 1))
#% for traceback
#Tback = zeros(r+1,c+1);
Tback = np.zeros((r + 1, c + 1))
#if True:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("dp3(): printing locally optimum solutions:")
# Alex: trying out to find a better solution than dp3() !!!!TODO : more
# This solution is basically the one returned by causal() IF we do NOT apply Matlab.filter2() on Vspace
for j in range(1, c + 1):
maxCol = 0.0
maxPos = -1
for i in range(1, r + 1):
assert D[i, j] >= 0.0
if maxCol < D[i, j]:
maxCol = D[i, j]
maxPos = i
# So for query frame j we have a candidate matching ref frame i
common.DebugPrint(
"dp3(): for query frame %d - candidate frame %d" %
(j - 1, maxPos))
common.DebugPrint(
"dp3(): for query frame %d we found matching ref frame %d" %
(j - 1, maxPos))
# !!!!TODO: make i =0.., j=0.. and substitute i-1 with i, i-2 with i-1, etc
#for i = 1:r;
for i in range(1, r + 1):
#for j = 1:c
for j in range(1, c + 1):
#if (i>1) && (j>1)
if (i > 1) and (j > 1):
#dd1 = w(1)*Vspace(i-1,max(1,j-2),1);
dd1 = w[0] * Vspace[i - 2, max(0, j - 3), 0]
#dd2 = w(1)*Vspace(max(1,i-2),j-1,1);
dd2 = w[0] * Vspace[max(0, i - 3), j - 2, 0]
#dd3 = w(1)*Vspace(i-1,j-1,1);
dd3 = w[0] * Vspace[i - 2, j - 2, 0]
#dd4 = w(1)*Vspace(i-1,j,1);
dd4 = w[0] * Vspace[i - 2, j - 1, 0]
#dd5 = w(1)*Vspace(i,j-1,1);
dd5 = w[0] * Vspace[i - 1, j - 2, 0]
if d > 1:
#for sc = 2:d
for sc in range(2, d + 1):
#dd1 = max(dd1, w(sc)*Vspace(i-1,max(1,j-2),sc));
dd1 = max(dd1, w[sc - 1] * \
Vspace[i - 2, max(0, j - 3), sc - 1])
#dd2 = max(dd2, w(sc)*Vspace(max(1,i-2),j-1,sc));
dd2 = max(dd2, w[sc - 1] * \
Vspace[max(0, i - 3), j - 2, sc - 1])
#dd3 = max(dd3, w(sc)*Vspace(i-1,j-1,sc));
dd3 = max(dd3,
w[sc - 1] * Vspace[i - 2, j - 2, sc - 1])
#dd4 = max(dd4, w(sc)*Vspace(i-1,j,sc));
dd4 = max(dd4,
w[sc - 1] * Vspace[i - 2, j - 1, sc - 1])
#dd5 = max(dd5, w(sc)*Vspace(i,j-1,sc));
dd5 = max(dd5,
w[sc - 1] * Vspace[i - 1, j - 2, sc - 1])
#D(i,j-1)=D(i,j-1)+dd1;
D[i - 1, j - 2] += dd1
#D(i-1,j)=D(i-1,j)+dd2;
D[i - 2, j - 1] += dd2
#D(i,j)=D(i,j)+ dd3;
D[i - 1, j - 1] += dd3
#D(i,j+1)=D(i,j+1)+ dd4;
D[i - 1, j] += dd4
#D(i+1,j)=D(i+1,j)+ dd5;
D[i, j - 1] += dd5
#% % instead of above loop, use the following five lines when scales=6
#% D(i,j-1)=D(i,j-1)+max([w(1)*Vspace(i-1,max(1,j-2),1),w(2)*Vspace(i-1,max(1,j-2),2),w(3)*Vspace(i-1,max(1,j-2),3),w(4)*Vspace(i-1,max(1,j-2),4),w(5)*Vspace(i-1,max(1,j-2),5),w(6)*Vspace(i-1,max(1,j-2),6)]);
#% D(i-1,j)=D(i-1,j)+max([w(1)*Vspace(max(1,i-2),j-1,1),w(2)*Vspace(max(1,i-2),j-1,2),w(3)*Vspace(max(1,i-2),j-1,3),w(4)*Vspace(max(1,i-2),j-1,4),w(5)*Vspace(max(1,i-2),j-1,5),w(6)*Vspace(max(1,i-2),j-1,6)]);
#% D(i,j)=D(i,j)+max([w(1)*Vspace(i-1,j-1,1),w(2)*Vspace(i-1,j-1,2),w(3)*Vspace(i-1,j-1,3),w(4)*Vspace(i-1,j-1,4),w(5)*Vspace(i-1,j-1,5),w(6)*Vspace(i-1,j-1,6)]);
#% D(i,j+1)=D(i,j+1)+max([w(1)*Vspace(i-1,j,1),w(2)*Vspace(i-1,j,2),w(3)*Vspace(i-1,j,3),w(4)*Vspace(i-1,j,4),w(5)*Vspace(i-1,j,5),w(6)*Vspace(i-1,j,6)]);
#% D(i+1,j)=D(i+1,j)+max([w(1)*Vspace(i,j-1,1),w(2)*Vspace(i,j-1,2),w(3)*Vspace(i,j-1,3),w(4)*Vspace(i,j-1,4),w(5)*Vspace(i,j-1,5),w(6)*Vspace(i,j-1,6)]);
#% [dmax, tb] = max([D(i, j), D(i-1, j), D(i, j-1), D(i,j+1),D(i+1,j)]);
#[dmax, tb] = max([D(i, j)+1/sqrt(2), D(i-1, j)+1/sqrt(5), D(i, j-1)+1/sqrt(5), D(i,j+1)+1, D(i+1,j)+1]);
dmax, tb = Matlab.max(np.array([ \
D[i - 1, j - 1] + 1.0 / math.sqrt(2.0), \
D[i - 2, j - 1] + 1.0 / math.sqrt(5.0), \
D[i - 1, j - 2] + 1.0 / math.sqrt(5.0), \
D[i - 1, j] + 1,
D[i, j - 1] + 1]))
else:
#[dmax, tb] = max([D(i,j), D(i,j+1), D(i+1,j)]);
dmax, tb = Matlab.max( \
np.array([D[i - 1, j - 1], D[i - 1, j], D[i, j - 1]]))
if common.MY_DEBUG_STDOUT:
common.DebugPrint("dp3(): dmax = %s" % str(dmax))
common.DebugPrint("dp3(): tb = %s" % str(tb))
#D(i+1,j+1) = D(i+1,j+1)+dmax;
if NEW_DP3_ALEX:
cost[i, j] = 0
#!!!!TODO: think more
else:
D[i, j] += dmax
#!!!!TODO: for me it's weird he adds dmax here...
#Tback(i+1,j+1) = tb;
Tback[i, j] = tb
if common.MY_DEBUG_STDOUT:
common.DebugPrint("dp3(): D.shape = %s" % str(D.shape))
common.DebugPrint("dp3(): D = %s" % str(D))
common.DebugPrint("dp3(): Tback.shape = %s" % str(Tback.shape))
common.DebugPrint("dp3(): Tback = %s" % str(Tback))
#% Traceback
i = r + 1
j = c + 1
y = i - 1
x = j - 1
#while i > 2 & j > 2
while (i > 2) and (j > 2):
#tb = Tback(i,j);
tb = Tback[i - 1, j - 1] + 1
# In Matlab, max returns indices from 1..
if tb == 1:
i -= 1
j -= 1
elif tb == 2:
i -= 2
j -= 1
elif tb == 3:
i -= 1
j -= 2
elif tb == 4:
i -= 1
j = j
elif tb == 5:
j -= 1
i = i
else:
#error;
assert False
#y = [i,y];
#NOT GOOD: y = np.c_[i, y];
y = np.hstack([i - 1, y])
#x = [j,x];
#NOT GOOD: x = np.c_[j, x];
x = np.hstack([j - 1, x])
if common.MY_DEBUG_STDOUT:
common.DebugPrint("dp3(): before D.shape = %s" % str(D.shape))
common.DebugPrint("dp3(): before D = %s" % str(D))
#% Strip off the edges of the D matrix before returning
#D = D(2:(r+1),2:(c+1));
D = D[1:(r + 1), 1:(c + 1)]
if common.MY_DEBUG_STDOUT:
common.DebugPrint("dp3(): D.shape = %s" % str(D.shape))
common.DebugPrint("dp3(): D = %s" % str(D))
#RD_start=str2num(RD(1).name(end-9:end-4));
RD_start = 1
if common.MY_DEBUG_STDOUT:
common.DebugPrint("dp3(): Vspace.shape = %s" % str(Vspace.shape))
#!!!!TODO: understand well what is x,y and why computes p
#for i=1:size(Vspace,2):
for i in range(0, Vspace.shape[1]):
#cross(i,1)=str2num(QD(i).name(end-9:end-4));
crossref[i, 0] = i
#i; #!!!!TODO: think if OK
#p=find(x==i);
p = np.nonzero(x == i)
p = p[0]
if common.MY_DEBUG_STDOUT:
common.DebugPrint("dp3(): x.shape = %s" % str(x.shape))
common.DebugPrint("dp3(): x = %s" % str(x))
common.DebugPrint("dp3(): y.shape = %s" % str(y.shape))
common.DebugPrint("dp3(): y = %s" % str(y))
common.DebugPrint("dp3(): i = %s" % str(i))
common.DebugPrint("dp3(): p = %s" % str(p))
#if isempty(p)
if p.size == 0:
#% Alex: Vali Codreanu said to change from temp=0; to temp=3;
#temp=3;
temp = 0
if common.MY_DEBUG_STDOUT:
common.DebugPrint("dp3(): temp = %s" % str(temp))
crossref[i, 1] = 0 + RD_start - 1
else:
#temp=y(p);
temp = y[p]
if common.MY_DEBUG_STDOUT:
common.DebugPrint("dp3(): temp = %s" % str(temp))
#cross(i,2)=temp(end)+RD_start-1;
if temp.size == 1:
# If temp has only 1 element:
crossref[i, 1] = temp + RD_start - 1
else:
crossref[i, 1] = temp[-1] + RD_start - 1
#assert temp.size == 1;
#common.DebugPrint("dp3(): temp = %s" % str(temp));
"""
#cross(i,2)=temp(end)+RD_start-1;
if True:
crossref[i, 1] = temp[-1] + RD_start - 1;
else:
# BAD IDEA!!!!TODO
crossref[i, 1] = temp + RD_start - 1;
"""
#printf("dp3(): Done in blabla secs\n");
common.DebugPrint("dp3(): crossref = %s" % str(crossref))
ComputeCost(crossref, VV, "crossref_dp3.txt")
#return [y,x,D,Tback,cross]
return y, x, D, Tback, crossref
def causal(Vspace, H, numFramesQ, numFramesR, BOV_flag, cropflag, const_type, nop=0):
#%cons_type : 1 means that spatial constraint applies after voting
#% 2 means that spatial constraint applies before voting -> NOT
#% USED IN PAPER
# causal() is the local/greedy optimization solution
#% nargin>7 means that we do the local smoothing with RANSAC (see the paper)
"""
common.DebugPrint("causal(): At entrance Vspace=%s,\nH=%s, \n" \
"numFramesQ=%s, numFramesR=%s, BOV_flag=%d, cropflag=%d, const_type=%d, " \
"nop=%d" % \
(str(Vspace), str(H), numFramesQ, numFramesR, BOV_flag, cropflag, \
const_type, nop));
"""
print (
"causal(): At entrance \n"
" numFramesQ=%s, numFramesR=%s, BOV_flag=%d, cropflag=%d, const_type=%d, "
" nop=%d" % (numFramesQ, numFramesR, BOV_flag, cropflag, const_type, nop)
)
# Normally BOV_flag=0, cropflag=0, const_type=1,nop=0
# if True:
if common.MY_DEBUG_STDOUT:
print ("causal(): Vspace.shape = %s" % str(Vspace.shape))
print ("causal(): H.shape = %s" % str(H.shape))
for i in range(Vspace.shape[2]):
common.DebugPrint("causal(): Vspace[:, :, %d] = \n%s" % (i, str(Vspace[:, :, i])))
"""
common.DebugPrint("causal(): Vspace[:, :, %d] = " % (i));
for r in range(Vspace.shape[0]):
for c in range(Vspace.shape[1]):
common.DebugPrint("%.4f " % Vspace[r, c, i]),
print;
"""
print
for i in range(H.shape[2]):
common.DebugPrint("causal(): H[:, :, %d] = \n%s" % (i, str(H[:, :, i])))
"""
common.DebugPrint("causal(): H[:, :, %d] = " % (i));
for r in range(H.shape[0]):
for c in range(H.shape[1]):
common.DebugPrint("%.4f " % H[r, c, i]),
print;
"""
# nos=size(Vspace,3); %3D matrix
nos = Vspace.shape[2]
#%3D matrix
#% same with the multi_scale_harris function
sigma_0 = 1.2
# n=0:nos-1;
n = range(0, nos)
# sigma_D=sqrt(1.8).^n*sigma_0; % We should see sqrt(1.8) returning a vector of nos elements
# NOT GOOD in Python: sigma_D = math.sqrt(1.8)**n * sigma_0;
sq18 = np.ones((1, nos)) * math.sqrt(1.8)
sigma_D = sq18 ** n * sigma_0
# w = sigma_D;
w = sigma_D[0]
# common.DebugPrint("causal(): w = %s" % str(w));
# common.DebugPrint("causal(): w.shape = %s" % str(w.shape));
w = w / w.sum()
# common.DebugPrint("causal(): w = %s" % str(w));
# Alex: normally NOT executed
if BOV_flag == 1:
# w=fliplr(w);
w = w[:, ::-1] # We flip on the vertical (left becomes right)
print ("causal(): w = %s" % str(w))
# Alex: normally NOT executed
if (const_type == 1) and (BOV_flag == 1):
assert False
# Normally this code does NOT get executed
# VV=zeros(size(Vspace,1),size(Vspace,2));
VV = np.zeros((Vspace.shape[0], Vspace.shape[1]))
for j in range(1, nos + 1):
# VV=VV+Vspace(:,:,j);
VV = VV + Vspace[:, :, j - 1]
# [X,Y]=sort(VV,'descend');
X, Y = sort(VV, "descend")
#%enable top-N list
#%N = 50;
#%N = 100;
N = 300
for s in range(1, nos + 1):
for i in range(1, Vspace.shape[1] + 1):
# y=Y(:,i);
y = Y[:, i - 1]
# votes=Vspace(:,i,s);
votes = Vspace[:, i - 1, s - 1]
# h=H(:,i,s);
h = H[:, i - 1, s - 1]
# votes(y(N+1:end))=0;
votes[y[N:]] = 0
# h(y(N+1:end))=0;
h[y[N:]] = 0
# Vspace(:,i,s)=votes;
Vspace[:, i - 1, s - 1] = votes
# H(:,i,s)=h;
H[:, i - 1, s - 1] = h
# clear VV
VV = None
# QD=dir([q_path 'multiharlocs*.mat']);
# RD=dir([r_path 'multiharlocs*.mat']);
# QD = [None] * numFramesQ;
# RD = [None] * numFramesR;
# cross=zeros(length(QD),2);
# crossref = np.zeros( (len(QD), 2) );
crossref = np.zeros((numFramesQ, 2))
# common.DebugPrint("causal(): crossref = %s" % str(crossref));
#% str=['load ' r_path 'multiscale_nod'];
#% eval(str)
#%replace nan votes with zeros
#%sum(sum(isnan(Vspace(:,:,1))))
# We transform nan in 0 in Vspace
"""
We substitute i - 1 with i, since array indexing starts from 1 in Matlab
and 0 in Python.
"""
# for i in range(1, nos + 1):
for i in range(nos):
# V_temp=Vspace(:,:,i);
# V_temp = Vspace[:, :, i - 1];
V_temp = Vspace[:, :, i]
# V_temp(isnan(V_temp))=0;
V_temp[np.isnan(V_temp)] = 0
# Vspace(:,:,i)=V_temp;
# Vspace[:, :, i - 1] = V_temp;
Vspace[:, :, i] = V_temp
# Alex: I personally find this idea of using filter2() VERY BAD - the results in Vspace should already be VERY good
if CAUSAL_DO_NOT_SMOOTH == False:
#% this filtering of votes favors smoother results
#%b=ones(11,1);
#%b=b/prod(size(b));
b = Matlab.hamming(11)
#% USED
# if False:
"""
We substitute i - 1 with i, since array indexing starts from 1 in Matlab
and 0 in Python.
"""
# for i in range(1, nos + 1):
for i in range(nos):
# Vspace(:,:,i)=filter2(b,Vspace(:,:,i));
"""
From the Matlab help:
Y = filter2(h,X) filters
the data in X with the two-dimensional FIR filter
in the matrix h. It computes the result, Y,
using two-dimensional correlation, and returns the central part of
the correlation that is the same size as X.
"""
# Vspace[:, :, i - 1] = Matlab.filter2(b, Vspace[:, :, i - 1]);
Vspace[:, :, i] = Matlab.filter2(b, Vspace[:, :, i])
#% use imfilter if filter2 takes time #!!!!TODO: do the optimization Evangelidis says here
"""
From Matlab help:
M = mean(A,dim) returns
the mean values for elements along the dimension of A specified
by scalar dim. For matrices, mean(A,2) is
a column vector containing the mean value of each row.
"""
# V=mean(Vspace,3); % this might help more instead of starting from zero votes
V = Vspace.mean(2) # this might help more instead of starting from zero votes
# common.DebugPrint("causal(): V.shape = %s" % str(V.shape));
"""
We substitute i - 1 with i, since array indexing starts from 1 in Matlab
and 0 in Python.
"""
# for i in range(1, nos + 1):
for i in range(nos):
if cropflag == 0:
if (const_type == 1) and (BOV_flag == 1):
# V=V+w(i)*H(:,:,i)+Vspace(:,:,i);
# V += w[i - 1] * H[:, :, i - 1] + Vspace[:, :, i - 1]; #!!!!TODO: think well *
V += w[i] * H[:, :, i] + Vspace[:, :, i]
#!!!!TODO: think well *
else:
# Alex: we are normally in this case, since cropflag == 0, const_type == 1, BOV_flag == 0
# V=V+w(i)*Vspace(:,:,i);
# V += w[i - 1] * Vspace[:, :, i - 1]; #!!!!TODO: think well * Exception "ValueError: operands could not be broadcast together with shapes (5) (23,8)"
V += w[i] * Vspace[:, :, i]
#!!!!TODO: think well * Exception "ValueError: operands could not be broadcast together with shapes (5) (23,8)"
else:
# V=V+w(i)*Vspace(:,:,i)+H(:,:,i);
# V += w[i - 1] * Vspace[:, :, i - 1] + H[:, :, i - 1]; #!!!!TODO: think well *
V += w[i] * Vspace[:, :, i] + H[:, :, i]
#!!!!TODO: think well *
if common.MY_DEBUG_STDOUT:
# common.DebugPrint("causal(): Vspace.shape = %s" % str(Vspace.shape));
common.DebugPrint("causal(): V.shape = %s" % str(V.shape))
common.DebugPrint("causal(): V (the matrix used to choose the max-voting reference frame) = %s" % str(V))
# common.DebugPrint("causal(): len(QD) = %d" % len(QD));
# for iFor in range(1, len(QD) + 1):
"""
We substitute iFor -1 with iFor since arrays start with 0 in Python,
not with 1 like in Matlab
"""
# for iFor in range(1, numFramesQ + 1):
for iFor in range(numFramesQ):
# cross(i,1)=str2num(QD(i).name(end-9:end-4));
crossref[iFor, 0] = iFor
# TODO - think well
# [a,b]=max(V(:,i));
b = V[:, iFor].argmax()
a = V[:, iFor][b]
# cross(i,2)=str2num(RD(b).name(end-9:end-4));
crossref[iFor, 1] = b
#!!!!TODO - think well
# We normally do NOT execute the following code, since nop == 0
# if nargin>7
if nop != 0:
# XX = cross(:,1)';
XX = crossref[:, 0].T
# YY = cross(:,2)';
YY = crossref[:, 1].T
YY_new = YY
if mod(nop, 2) == 0:
# error('Number of points (parameter NOP) must be odd number');
# common.DebugPrint("Number of points (parameter NOP) must be odd number");
quit()
# miso is used only as array index and in range expressions --> it can be an integer
miso = (nop - 1) / 2
if nop < 7:
# common.DebugPrint("Choose more than 5 points for local fitting");
pass
# for i = miso+1:length(XX)-miso
for iFor in range(miso + 1, len(XX) - miso + 1):
# xx = XX(i-miso:i+miso);
xx = XX[iFor - miso - 1 : iFor + miso]
# yy = YY(i-miso:i+miso);
yy = YY[iFor - miso - 1 : iFor + miso]
if nop < 9:
"""
iterNum is used only in range expressions and
array dimensions --> it can be an integer.
"""
iterNum = nop * (nop - 1) / 2
else:
iterNum = 10
thDist = 2
thInlrRatio = 0.6
"""
From Matlab help:
RANSAC Use RANdom SAmple Consensus to fit a line
RESCOEF = RANSAC(PTS,ITERNUM,THDIST,THINLRRATIO) PTS is 2*n matrix including
n points, ITERNUM is the number of iteration, THDIST is the inlier
distance threshold and ROUND(THINLRRATIO*SIZE(PTS,2)) is the inlier number threshold. The final
fitted line is y = alpha*x+beta.
Yan Ke @ THUEE, [email protected]
"""
# [ alpha, beta ] = ransac_line( [xx;yy],iterNum,thDist,thInlrRatio );
alpha, beta = ransac_line(np.r_[xx, yy], iterNum, thDist, thInlrRatio)
if alpha != 0:
yy_new = alpha * xx + beta
#!!!!TODO: think well *
YY_new[iFor - 1] = yy_new[miso + 1 - 1]
# cross(:,2) = YY_new;
crossref[:, 2] = YY_new
crossref = crossref.astype(int)
print ("causal(): crossref = %s" % str(crossref))
ComputeCost(crossref, V, "crossref_causal.txt")
if False: # True:
# print("Am here1111");
# causal_Alex(Vspace, numFramesQ, numFramesR);
# print("Am here2222");
# dp3(Vspace, numFramesR, numFramesQ, BOV_flag);
# print("Am here3333");
# dp_Alex(Vspace, numFramesR, numFramesQ, BOV_flag);
dp_Alex(Vspace, numFramesR, numFramesQ, BOV_flag, PREV_REF=5, NEXT_REF=-1)
return crossref
def causal(Vspace,
H,
numFramesQ,
numFramesR,
BOV_flag,
cropflag,
const_type,
nop=0):
#%cons_type : 1 means that spatial constraint applies after voting
#% 2 means that spatial constraint applies before voting -> NOT
#% USED IN PAPER
# causal() is the local/greedy optimization solution
#% nargin>7 means that we do the local smoothing with RANSAC (see the paper)
"""
common.DebugPrint("causal(): At entrance Vspace=%s,\nH=%s, \n" \
"numFramesQ=%s, numFramesR=%s, BOV_flag=%d, cropflag=%d, const_type=%d, " \
"nop=%d" % \
(str(Vspace), str(H), numFramesQ, numFramesR, BOV_flag, cropflag, \
const_type, nop));
"""
print("causal(): At entrance \n" \
" numFramesQ=%s, numFramesR=%s, BOV_flag=%d, cropflag=%d, const_type=%d, " \
" nop=%d" % \
(numFramesQ, numFramesR, BOV_flag, cropflag, \
const_type, nop))
# Normally BOV_flag=0, cropflag=0, const_type=1,nop=0
#if True:
if common.MY_DEBUG_STDOUT:
print("causal(): Vspace.shape = %s" % str(Vspace.shape))
print("causal(): H.shape = %s" % str(H.shape))
for i in range(Vspace.shape[2]):
common.DebugPrint("causal(): Vspace[:, :, %d] = \n%s" %
(i, str(Vspace[:, :, i])))
"""
common.DebugPrint("causal(): Vspace[:, :, %d] = " % (i));
for r in range(Vspace.shape[0]):
for c in range(Vspace.shape[1]):
common.DebugPrint("%.4f " % Vspace[r, c, i]),
print;
"""
print
for i in range(H.shape[2]):
common.DebugPrint("causal(): H[:, :, %d] = \n%s" %
(i, str(H[:, :, i])))
"""
common.DebugPrint("causal(): H[:, :, %d] = " % (i));
for r in range(H.shape[0]):
for c in range(H.shape[1]):
common.DebugPrint("%.4f " % H[r, c, i]),
print;
"""
#nos=size(Vspace,3); %3D matrix
nos = Vspace.shape[2]
#%3D matrix
#% same with the multi_scale_harris function
sigma_0 = 1.2
#n=0:nos-1;
n = range(0, nos)
#sigma_D=sqrt(1.8).^n*sigma_0; % We should see sqrt(1.8) returning a vector of nos elements
#NOT GOOD in Python: sigma_D = math.sqrt(1.8)**n * sigma_0;
sq18 = np.ones((1, nos)) * math.sqrt(1.8)
sigma_D = sq18**n * sigma_0
#w = sigma_D;
w = sigma_D[0]
#common.DebugPrint("causal(): w = %s" % str(w));
#common.DebugPrint("causal(): w.shape = %s" % str(w.shape));
w = w / w.sum()
#common.DebugPrint("causal(): w = %s" % str(w));
# Alex: normally NOT executed
if BOV_flag == 1:
#w=fliplr(w);
w = w[:, ::-1] # We flip on the vertical (left becomes right)
print("causal(): w = %s" % str(w))
# Alex: normally NOT executed
if (const_type == 1) and (BOV_flag == 1):
assert False
# Normally this code does NOT get executed
#VV=zeros(size(Vspace,1),size(Vspace,2));
VV = np.zeros((Vspace.shape[0], Vspace.shape[1]))
for j in range(1, nos + 1):
#VV=VV+Vspace(:,:,j);
VV = VV + Vspace[:, :, j - 1]
#[X,Y]=sort(VV,'descend');
X, Y = sort(VV, "descend")
#%enable top-N list
#%N = 50;
#%N = 100;
N = 300
for s in range(1, nos + 1):
for i in range(1, Vspace.shape[1] + 1):
#y=Y(:,i);
y = Y[:, i - 1]
#votes=Vspace(:,i,s);
votes = Vspace[:, i - 1, s - 1]
#h=H(:,i,s);
h = H[:, i - 1, s - 1]
#votes(y(N+1:end))=0;
votes[y[N:]] = 0
#h(y(N+1:end))=0;
h[y[N:]] = 0
#Vspace(:,i,s)=votes;
Vspace[:, i - 1, s - 1] = votes
#H(:,i,s)=h;
H[:, i - 1, s - 1] = h
#clear VV
VV = None
#QD=dir([q_path 'multiharlocs*.mat']);
#RD=dir([r_path 'multiharlocs*.mat']);
#QD = [None] * numFramesQ;
#RD = [None] * numFramesR;
#cross=zeros(length(QD),2);
#crossref = np.zeros( (len(QD), 2) );
crossref = np.zeros((numFramesQ, 2))
#common.DebugPrint("causal(): crossref = %s" % str(crossref));
#% str=['load ' r_path 'multiscale_nod'];
#% eval(str)
#%replace nan votes with zeros
#%sum(sum(isnan(Vspace(:,:,1))))
# We transform nan in 0 in Vspace
"""
We substitute i - 1 with i, since array indexing starts from 1 in Matlab
and 0 in Python.
"""
#for i in range(1, nos + 1):
for i in range(nos):
#V_temp=Vspace(:,:,i);
#V_temp = Vspace[:, :, i - 1];
V_temp = Vspace[:, :, i]
#V_temp(isnan(V_temp))=0;
V_temp[np.isnan(V_temp)] = 0
#Vspace(:,:,i)=V_temp;
#Vspace[:, :, i - 1] = V_temp;
Vspace[:, :, i] = V_temp
# Alex: I personally find this idea of using filter2() VERY BAD - the results in Vspace should already be VERY good
if CAUSAL_DO_NOT_SMOOTH == False:
#% this filtering of votes favors smoother results
#%b=ones(11,1);
#%b=b/prod(size(b));
b = Matlab.hamming(11)
#% USED
#if False:
"""
We substitute i - 1 with i, since array indexing starts from 1 in Matlab
and 0 in Python.
"""
#for i in range(1, nos + 1):
for i in range(nos):
#Vspace(:,:,i)=filter2(b,Vspace(:,:,i));
"""
From the Matlab help:
Y = filter2(h,X) filters
the data in X with the two-dimensional FIR filter
in the matrix h. It computes the result, Y,
using two-dimensional correlation, and returns the central part of
the correlation that is the same size as X.
"""
#Vspace[:, :, i - 1] = Matlab.filter2(b, Vspace[:, :, i - 1]);
Vspace[:, :, i] = Matlab.filter2(b, Vspace[:, :, i])
#% use imfilter if filter2 takes time #!!!!TODO: do the optimization Evangelidis says here
"""
From Matlab help:
M = mean(A,dim) returns
the mean values for elements along the dimension of A specified
by scalar dim. For matrices, mean(A,2) is
a column vector containing the mean value of each row.
"""
#V=mean(Vspace,3); % this might help more instead of starting from zero votes
V = Vspace.mean(
2) # this might help more instead of starting from zero votes
#common.DebugPrint("causal(): V.shape = %s" % str(V.shape));
"""
We substitute i - 1 with i, since array indexing starts from 1 in Matlab
and 0 in Python.
"""
#for i in range(1, nos + 1):
for i in range(nos):
if cropflag == 0:
if (const_type == 1) and (BOV_flag == 1):
#V=V+w(i)*H(:,:,i)+Vspace(:,:,i);
#V += w[i - 1] * H[:, :, i - 1] + Vspace[:, :, i - 1]; #!!!!TODO: think well *
V += w[i] * H[:, :, i] + Vspace[:, :, i]
#!!!!TODO: think well *
else:
# Alex: we are normally in this case, since cropflag == 0, const_type == 1, BOV_flag == 0
#V=V+w(i)*Vspace(:,:,i);
#V += w[i - 1] * Vspace[:, :, i - 1]; #!!!!TODO: think well * Exception "ValueError: operands could not be broadcast together with shapes (5) (23,8)"
V += w[i] * Vspace[:, :, i]
#!!!!TODO: think well * Exception "ValueError: operands could not be broadcast together with shapes (5) (23,8)"
else:
#V=V+w(i)*Vspace(:,:,i)+H(:,:,i);
#V += w[i - 1] * Vspace[:, :, i - 1] + H[:, :, i - 1]; #!!!!TODO: think well *
V += w[i] * Vspace[:, :, i] + H[:, :, i]
#!!!!TODO: think well *
if common.MY_DEBUG_STDOUT:
#common.DebugPrint("causal(): Vspace.shape = %s" % str(Vspace.shape));
common.DebugPrint("causal(): V.shape = %s" % str(V.shape))
common.DebugPrint(
"causal(): V (the matrix used to choose the max-voting reference frame) = %s"
% str(V))
#common.DebugPrint("causal(): len(QD) = %d" % len(QD));
#for iFor in range(1, len(QD) + 1):
"""
We substitute iFor -1 with iFor since arrays start with 0 in Python,
not with 1 like in Matlab
"""
#for iFor in range(1, numFramesQ + 1):
for iFor in range(numFramesQ):
#cross(i,1)=str2num(QD(i).name(end-9:end-4));
crossref[iFor, 0] = iFor
#TODO - think well
#[a,b]=max(V(:,i));
b = V[:, iFor].argmax()
a = V[:, iFor][b]
#cross(i,2)=str2num(RD(b).name(end-9:end-4));
crossref[iFor, 1] = b
#!!!!TODO - think well
# We normally do NOT execute the following code, since nop == 0
#if nargin>7
if nop != 0:
#XX = cross(:,1)';
XX = crossref[:, 0].T
#YY = cross(:,2)';
YY = crossref[:, 1].T
YY_new = YY
if mod(nop, 2) == 0:
#error('Number of points (parameter NOP) must be odd number');
#common.DebugPrint("Number of points (parameter NOP) must be odd number");
quit()
# miso is used only as array index and in range expressions --> it can be an integer
miso = (nop - 1) / 2
if nop < 7:
#common.DebugPrint("Choose more than 5 points for local fitting");
pass
#for i = miso+1:length(XX)-miso
for iFor in range(miso + 1, len(XX) - miso + 1):
#xx = XX(i-miso:i+miso);
xx = XX[iFor - miso - 1:iFor + miso]
#yy = YY(i-miso:i+miso);
yy = YY[iFor - miso - 1:iFor + miso]
if nop < 9:
"""
iterNum is used only in range expressions and
array dimensions --> it can be an integer.
"""
iterNum = nop * (nop - 1) / 2
else:
iterNum = 10
thDist = 2
thInlrRatio = 0.6
"""
From Matlab help:
RANSAC Use RANdom SAmple Consensus to fit a line
RESCOEF = RANSAC(PTS,ITERNUM,THDIST,THINLRRATIO) PTS is 2*n matrix including
n points, ITERNUM is the number of iteration, THDIST is the inlier
distance threshold and ROUND(THINLRRATIO*SIZE(PTS,2)) is the inlier number threshold. The final
fitted line is y = alpha*x+beta.
Yan Ke @ THUEE, [email protected]
"""
#[ alpha, beta ] = ransac_line( [xx;yy],iterNum,thDist,thInlrRatio );
alpha, beta = ransac_line( np.r_[xx, yy], iterNum, \
thDist, thInlrRatio )
if alpha != 0:
yy_new = alpha * xx + beta
#!!!!TODO: think well *
YY_new[iFor - 1] = yy_new[miso + 1 - 1]
#cross(:,2) = YY_new;
crossref[:, 2] = YY_new
crossref = crossref.astype(int)
print("causal(): crossref = %s" % str(crossref))
ComputeCost(crossref, V, "crossref_causal.txt")
if False: #True:
#print("Am here1111");
#causal_Alex(Vspace, numFramesQ, numFramesR);
#print("Am here2222");
#dp3(Vspace, numFramesR, numFramesQ, BOV_flag);
#print("Am here3333");
#dp_Alex(Vspace, numFramesR, numFramesQ, BOV_flag);
dp_Alex(Vspace,
numFramesR,
numFramesQ,
BOV_flag,
PREV_REF=5,
NEXT_REF=-1)
return crossref
def MyImageReadMSH(index):
global g;
"""
common.DebugPrint("MyImageReadMSH(): initFrame = %d)" % \
(config.initFrame[g.indexVideo]));
index += config.initFrame[g.indexVideo];
"""
"""
common.DebugPrint("Entered MyImageReadMSH(capture=%s, index=%s)" % \
(str(capture), str(index)));
"""
common.DebugPrint("Entered MyImageReadMSH(index=%s)" % \
(str(index)));
"""
We must reopen the capture device in each different process, otherwise the
program blocks at the first operation in the "global" capture device.
"""
if g.harlocsFolder.endswith(config.HARRIS_QUERY_FOLDER_NAME):
if g.captureQ == None:
capture = cv2.VideoCapture(sys.argv[1]);
g.captureQ = capture;
common.DebugPrint("MyImageReadMSH(): new capture=%s" % \
(str(capture)));
else:
capture = g.captureQ;
elif g.harlocsFolder.endswith(config.HARRIS_REFERENCE_FOLDER_NAME):
if g.captureR == None:
capture = cv2.VideoCapture(sys.argv[2]);
g.captureR = capture;
common.DebugPrint("MyImageReadMSH(): new capture=%s" % \
(str(capture)));
else:
capture = g.captureR;
else:
assert False;
assert (g.indexVideo == 0) or (g.indexVideo == 1);
if config.OCV_OLD_PY_BINDINGS:
capture.set(cv2.cv.CV_CAP_PROP_POS_FRAMES, \
index);
else:
"""
From http://docs.opencv.org/modules/highgui/doc/reading_and_writing_images_and_video.html#videocapture-get:
<<CV_CAP_PROP_POS_FRAMES 0-based index of the frame to be
decoded/captured next.>>
"""
capture.set(cv2.CAP_PROP_POS_FRAMES, \
index);
common.DebugPrint("MyImageReadMSH(): after capture.set()");
# This is only for (paranoid) testing purposes:
if config.OCV_OLD_PY_BINDINGS:
indexCrt = capture.get(cv2.cv.CV_CAP_PROP_POS_FRAMES);
else:
"""
From http://docs.opencv.org/modules/highgui/doc/reading_and_writing_images_and_video.html#videocapture-get:
<<CV_CAP_PROP_POS_FRAMES 0-based index of the frame to be
decoded/captured next.>>
"""
indexCrt = capture.get(cv2.CAP_PROP_POS_FRAMES);
#assert int(indexCrt) == index;
#!!!!TODO: think if OK
if int(indexCrt) != index:
common.DebugPrint( \
"MyImageReadMSH(): indexCrt != index --> returning black frame");
ret = False;
else:
#common.DebugPrint("Alex: frameR = %d" % frameR);
#if myIndex > numFramesR:
# break;
#ret, img = r_path.read();
ret, img = capture.read();
#if ret == False:
# break;
#!!!!TODO: think if well
#assert ret == True;
if ret == False:
common.DebugPrint(
"MyImageReadMSH(index=%d): ret == False --> returning None" % index);
img = None;
else:
common.DebugPrint("MyImageReadMSH(): img.shape = %s" % str(img.shape));
common.DebugPrint("MyImageReadMSH(): img.dtype = %s" % str(img.dtype));
#!!!!TODO: I suggest to do the gray conversion at reading, not in multi_scale_harris.py
if False:
# In the Matlab code he reads gray/8bpp JPEGs
imgGray = common.ConvertImgToGrayscale(img);
if config.VIDEO_FRAME_RESIZE_SCALING_FACTOR != 1:
# We resize the image
img = Matlab.imresize(img, \
scale=config.VIDEO_FRAME_RESIZE_SCALING_FACTOR);
common.DebugPrint("Exiting MyImageReadMSH()");
if False:
return imgGray;
return img;
def ComputeHarlocs(capture, counterStep, folderName, fileNamePrefix,
fileNameExtension=".csv", indexVideo=-1):
print( \
"Entered ComputeHarlocs(capture=%s, counterStep=%d, folderName=%s, " \
"indexVideo=%d)" % \
(str(capture), counterStep, folderName, indexVideo));
harlocsFolder = config.VIDEOS_FOLDER + "/" + folderName;
t1 = float(cv2.getTickCount());
harlocs = [];
if config.OCV_OLD_PY_BINDINGS:
numFrames = int(capture.get(cv2.cv.CV_CAP_PROP_FRAME_COUNT));
else:
numFrames = int(capture.get(cv2.CAP_PROP_FRAME_COUNT));
common.DebugPrint("ComputeHarlocs(): numFrames = %d" % numFrames);
if not os.path.exists(harlocsFolder):
os.makedirs(harlocsFolder);
else:
#!!!!TODO: check that the loaded Harlocs are complete - same frame numbers as in the videos
# Folder with precomputed Harris features exists
folderContent = os.listdir(harlocsFolder);
sortedFolderContent = sorted(folderContent);
for fileName in sortedFolderContent:
pathFileName = harlocsFolder + "/" + fileName;
"""
common.DebugPrint("ComputeHarlocs(): pathFileName = %s" % pathFileName);
common.DebugPrint("ComputeHarlocs(): fileName = %s" % fileName);
"""
if os.path.isfile(pathFileName) and \
fileName.startswith(fileNamePrefix) and \
pathFileName.endswith(fileNameExtension):
common.DebugPrint("ComputeHarlocs(): Loading %s" % pathFileName);
harrisFeatures = multi_scale_harris.LoadMultiScaleHarrisFeatures(pathFileName);
harlocs.append(harrisFeatures);
if config.endFrame[indexVideo] == -1:
assert (len(harlocs) + config.initFrame[indexVideo]) == numFrames; #!!!!TODO: if condition is NOT met, give a nicer error, or redo computations of Harlocs
else:
assert (len(harlocs) + config.initFrame[indexVideo]) == config.endFrame[indexVideo] + 1; #!!!!TODO: if condition is NOT met, give a nicer error, or redo computations of Harlocs
return harlocs;
if config.USE_MULTITHREADING == True:
global g;
g.captureQ = None; # We need to reopen the capture device in each process, separately
g.captureR = None; # We need to reopen the capture device in each process, separately
#g.capture = capture;
g.harlocsFolder = harlocsFolder;
g.fileNamePrefix = fileNamePrefix;
g.fileNameExtension = fileNameExtension;
g.indexVideo = indexVideo;
if config.OCV_OLD_PY_BINDINGS:
frameCount = int(capture.get(cv2.cv.CV_CAP_PROP_FRAME_COUNT));
else:
frameCount = int(capture.get(cv2.CAP_PROP_FRAME_COUNT));
#listParams = range(frameCount);
listParams = range(config.initFrame[indexVideo], frameCount, counterStep);
common.DebugPrint("ComputeHarlocs(): frameCount = %d" % frameCount);
print("ComputeHarlocs(): Spawning a pool of %d workers" % \
config.numProcesses);
"""
# DEBUG purposes ONLY - since when we use Pool() and call function, if
# we have an error in the function the exception reported is very
# vague...
for i in listParams:
IterationStandaloneMSH(i);
#import time
#time.sleep(1000);
"""
"""
Start worker processes to use on multi-core processor (circumvent
also the GIL issue).
"""
pool = multiprocessing.Pool(processes=config.numProcesses);
print("ComputeHarlocs(): Spawned a pool of %d workers" % \
config.numProcesses);
#res = pool.map(IterationStandaloneMSH, listParams);
# See https://docs.python.org/2/library/multiprocessing.html#module-multiprocessing.pool
res = pool.map(func=IterationStandaloneMSH, iterable=listParams, \
chunksize=1);
print("Pool.map returns %s" % str(res));
"""
From https://medium.com/building-things-on-the-internet/40e9b2b36148
close the pool and wait for the work to finish
"""
pool.close();
pool.join();
#!!!!TODO: do more efficient - don't load the results from the CSV files
return ComputeHarlocs(capture, counterStep, folderName, \
fileNamePrefix, fileNameExtension, indexVideo);
#return [];
#indexHarloc = 0;
while True:
if config.OCV_OLD_PY_BINDINGS:
framePos = capture.get(cv2.cv.CV_CAP_PROP_POS_FRAMES);
else:
"""
From http://docs.opencv.org/modules/highgui/doc/reading_and_writing_images_and_video.html#videocapture-get:
<<CV_CAP_PROP_POS_FRAMES 0-based index of the frame to be decoded/captured next.>>
"""
framePos = capture.get(cv2.CAP_PROP_POS_FRAMES);
common.DebugPrint("ComputeHarlocs(): framePos = %d" % framePos);
counter = int(framePos); #0
common.DebugPrint("ComputeHarlocs(): counter = %d" % counter);
ret, img = capture.read();
if common.MY_DEBUG_STDOUT:
common.DebugPrint("ComputeHarlocs(): img = %s" % str(img));
if False and config.SAVE_FRAMES:
fileName = config.IMAGES_FOLDER + "/img_%05d.png" % counter;
if not os.path.exists(fileName):
#print "dir(img) = %s"% str(dir(img))
"""
imgCV = cv.fromarray(img)
cv2.imwrite(fileName, imgCV)
"""
cv2.imwrite(fileName, img);
#if ret == False: #MatchFrames.counterQ == 3:
if (ret == False) or ((counter > numFrames) or \
(config.endFrame[indexVideo] != -1 and \
counter > config.endFrame[indexVideo])):
break;
if config.VIDEO_FRAME_RESIZE_SCALING_FACTOR != 1:
img = Matlab.imresize(img, \
scale=config.VIDEO_FRAME_RESIZE_SCALING_FACTOR);
im = img;
pp = multi_scale_harris.multi_scale_harris(im, nos, disp=0); # n=0:nos-1
#harlocs = pp
harlocs.append(pp);
multi_scale_harris.StoreMultiScaleHarrisFeatures( \
harlocsFolder + "/" + fileNamePrefix + "%05d%s" % \
(counter, fileNameExtension),
pp);
counter += counterStep;
# If we try to seek to a frame out-of-bounds frame it gets to the last one
if config.OCV_OLD_PY_BINDINGS:
capture.set(cv2.cv.CV_CAP_PROP_POS_FRAMES, counter);
else:
capture.set(cv2.CAP_PROP_POS_FRAMES, counter);
#indexHarloc += 1;
t2 = float(cv2.getTickCount());
myTime = (t2 - t1) / cv2.getTickFrequency();
common.DebugPrint("ComputeHarlocs(): computing the multiscale harlocs " \
"took %.6f [sec]" % myTime);
#common.DebugPrint("ComputeHarlocs(): len(harlocs) = %s" % str(len(harlocs)));
if False:
for i, h in enumerate(harlocs):
#common.DebugPrint("ComputeHarlocs(): len(harlocs[%d]) = %d" % \
# (i, len(h)));
multi_scale_harris.StoreMultiScaleHarrisFeatures(
harlocsFolder + "/" + fileNamePrefix + "%05d.txt" % i, h);
#if False:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("ComputeHarlocs(): harlocs = %s" % str(harlocs));
return harlocs;
def multi_scale_harris_Evangelidis(im, nos, disp):
common.DebugPrint("Entered multi_scale_harris_Evangelidis(nos=%d, disp=%s)" % \
(nos, str(disp)))
#tic
#if size(im, 3) == 3:
# im = rgb2gray(im)
tMSH1 = float(cv2.getTickCount())
if im.ndim == 3:
im = common.ConvertImgToGrayscale(im)
#im = im2double(im)
# From https://stackoverflow.com/questions/10873824/how-to-convert-2d-float-numpy-array-to-2d-int-numpy-array:
#DO NOT USE: im = im.astype(np.uint8); # This messes up tremendously the computation of harlocs
#im = im.astype(float);
im = im.astype(np.float32)
#!!!!COR is an UNused variable
#COR = zeros(size(im,1), size(im,2), size(im,3))
#COR = np.zeros((im.shape[0], im.shape[1], im.shape[1]));
# scale values
sigma_0 = 1.2
#n=[0:nos-1]; %scale levels
n = np.array(range(nos))
#scale levels
#sigma_D=sqrt(1.8).^n*sigma_0
sigma_D = math.sqrt(1.8)**n * sigma_0
#points=[];
points = np.array([])
#scn=sigma_D.^4;
scn = sigma_D**4
#for i=1:length(sigma_D)
#for i in range(1, len(sigma_D) + 1):
for i in range(1, nos + 1):
common.DebugPrint("multi_scale_harris_Evangelidis(): i = %s" %
(str(i)))
#sd=sigma_D(i); %differentiation (local) scale
sd = sigma_D[i - 1]
#%differentiation (local) scale
#si=sd/.5; %integration scale
si = sd / 0.5
#integration scale
w = 3 * sd
# size for gaussian kernel to compute derivatives: 3 times local_scale
r_w = int(round(w))
common.DebugPrint("multi_scale_harris_Evangelidis(): r_w = %s" % \
(str(r_w)))
#if mod(round(w),2):
if (r_w % 2) == 1:
#[xco,yco] = meshgrid(-round(w):round(w),-round(w):round(w))
mRange = range(-r_w, r_w + 1)
xco, yco = Matlab.meshgrid(mRange, mRange)
else:
#[xco,yco] = meshgrid(-(round(w)+1):round(w)+1,-(round(w)+1):round(w)+1)
mRange = range(-(r_w + 1), r_w + 2)
xco, yco = Matlab.meshgrid(mRange, mRange)
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): xco = %s" % \
str(xco))
common.DebugPrint("multi_scale_harris_Evangelidis(): yco = %s" % \
str(yco))
# Note: even for HD frames, xco.shape = (11, 11) (yco the same)
#arg = -(xco.*xco + yco.*yco) / (2*sd*sd)
#arg = -(xco * xco + yco * yco) / (2.0 * sd * sd);
arg = -(xco**2 + yco**2) / (2.0 * sd * sd)
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): arg = %s" % \
str(arg))
#%2d gaussian kernel
"""
From http://www.mathworks.com/help/matlab/ref/exp.html:
"exp(X) returns the exponential for each element of array X."
"""
#g=exp(arg)/(2*pi*sd^2); #2d gaussian kernel
g = np.exp(arg) / (2.0 * math.pi * pow(sd, 2))
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): g = %s" % \
str(g))
# normalize to suppress any gain
#if sum(g(:))~=0:
g_sum = g.sum()
#if abs(g.sum()) > 1.0e-6:
if abs(g_sum) > 1.0e-6:
#g = g / sum(g(:));
#g = g / float(g.sum());
g /= float(g_sum)
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): sd = %s" %
str(sd))
common.DebugPrint("multi_scale_harris_Evangelidis(): w = %s" %
str(w))
"""
#%Instead of computing derivatives in the filtered image,
we filter the image with the derivatives of the kernel.
"""
#% kernels for gaussian derivatives
#gx=-xco.*g/(sd*sd);
gx = -xco * g / float(sd * sd)
#gy=-yco.*g/(sd*sd);
gy = -yco * g / float(sd * sd)
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): gx = %s" % \
str(gx))
common.DebugPrint("multi_scale_harris_Evangelidis(): gy = %s" % \
str(gy))
"""
multi_scale_harris_Evangelidis(): arg.shape = (11, 11)
multi_scale_harris_Evangelidis(): g.shape = (11, 11)
multi_scale_harris_Evangelidis(): gx.shape = (11, 11)
multi_scale_harris_Evangelidis(): gy.shape = (11, 11)
multi_scale_harris_Evangelidis(): gi.shape = (15, 15)
"""
common.DebugPrint("multi_scale_harris_Evangelidis(): xco.shape = %s" %
str(xco.shape))
common.DebugPrint("multi_scale_harris_Evangelidis(): yco.shape = %s" %
str(yco.shape))
common.DebugPrint("multi_scale_harris_Evangelidis(): arg.shape = %s" %
str(arg.shape))
common.DebugPrint("multi_scale_harris_Evangelidis(): g.shape = %s" %
str(g.shape))
common.DebugPrint("multi_scale_harris_Evangelidis(): gx.shape = %s" %
str(gx.shape))
common.DebugPrint("multi_scale_harris_Evangelidis(): gy.shape = %s" %
str(gy.shape))
#%compute the derivatives
#Ix = imfilter(im, gx, 'replicate');
Ix = Matlab.imfilter(im, gx, "replicate")
#Iy = imfilter(im, gy, 'replicate');
Iy = Matlab.imfilter(im, gy, "replicate")
#% Alex: Ix and Iy have the same size as im
#if True:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): Ix = %s" % \
str(Ix))
common.DebugPrint("multi_scale_harris_Evangelidis(): Iy = %s" % \
str(Iy))
#% gaussian kernel to compute 2nd moment matrix
#if mod(floor(6*si),2) %size: six times the integration scale
if int(math.floor(
6 * si)) % 2 == 1: #size: six times the integration scale
#gi = fspecial('ga',max(1,fix(6*si)), si)
gi = Matlab.fspecial("ga", max(1, Matlab.fix(6 * si)), si)
else:
#gi = fspecial('ga',max(1,fix(6*si)+1), si)
gi = Matlab.fspecial("ga", max(1,
Matlab.fix(6 * si) + 1), si)
#if True:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): gi = %s" % \
str(gi))
common.DebugPrint("multi_scale_harris_Evangelidis(): gi.shape = %s" % \
str(gi.shape))
#Ix2 = imfilter(Ix.^2, gi, 'replicate');
Ix2 = Matlab.imfilter(Ix**2, gi, "replicate")
#Iy2 = imfilter(Iy.^2, gi, 'replicate');
Iy2 = Matlab.imfilter(Iy**2, gi, "replicate")
#Ixy = imfilter(Ix.*Iy, gi, 'replicate');
Ixy = Matlab.imfilter(Ix * Iy, gi, "replicate")
#% Alex: Ix2, Iy2 and Ixy have the same size as im
#if True:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): Ix2 = %s" % \
str(Ix2))
common.DebugPrint("multi_scale_harris_Evangelidis(): Iy2 = %s" % \
str(Iy2))
common.DebugPrint("multi_scale_harris_Evangelidis(): Ixy = %s" % \
str(Ixy))
common.DebugPrint("multi_scale_harris_Evangelidis(): Ix2.dtype = %s" % \
str(Ix2.dtype))
common.DebugPrint("multi_scale_harris_Evangelidis(): Iy2.dtype = %s" % \
str(Iy2.dtype))
common.DebugPrint("multi_scale_harris_Evangelidis(): Ixy.dtype = %s" % \
str(Ixy.dtype))
#%% Cornerness measure
#% Noble measure.
#%
#% M = (Ix2.*Iy2 - Ixy.^2)./(Ix2 + Iy2 + eps);
#% Harris measure.
#M = (Ix2.*Iy2 - Ixy.^2) - .06*(Ix2 + Iy2).^2;
M = (Ix2 * Iy2 - Ixy**2) - 0.06 * (Ix2 + Iy2)**2
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): M.dtype = %s" % \
str(M.dtype))
common.DebugPrint("multi_scale_harris_Evangelidis(): M = %s" % \
str(M))
#% Alex: scn is a vector - see definition above
#% Alex: M has the same size as im
#M = scn(i)*M;
M = scn[i - 1] * M
#thresh = 0.001*max(M(:));
thresh = 0.001 * M.max()
#% imagesc(M==abs(M));axis on;
#% colorbar
#% pause
"""
%keep points that are the maximum in a neighborhood of
radius=round(3*si/2) and are above thresh
%non-maximum supression and subpixel refinement
"""
#[r,c, rsubp, csubp] = my_nms(M, round(3*si/2), thresh);
r, c, rsubp, csubp = my_nms(M, round(3 * si / 2.0), thresh)
if common.MY_DEBUG_STDOUT:
common.DebugPrint(
"multi_scale_harris_Evangelidis(): r.shape = %s" %
str(r.shape))
common.DebugPrint("multi_scale_harris_Evangelidis(): r = %s" %
str(r))
common.DebugPrint(
"multi_scale_harris_Evangelidis(): c.shape = %s" %
str(c.shape))
common.DebugPrint("multi_scale_harris_Evangelidis(): c = %s" %
str(c))
common.DebugPrint(
"multi_scale_harris_Evangelidis(): rsubp.shape = %s" %
str(rsubp.shape))
common.DebugPrint("multi_scale_harris_Evangelidis(): rsubp = %s" %
str(rsubp))
common.DebugPrint(
"multi_scale_harris_Evangelidis(): csubp.shape = %s" %
str(csubp.shape))
common.DebugPrint("multi_scale_harris_Evangelidis(): csubp = %s" %
str(csubp))
#% Alex: r,c, rsubp, csubp seem to always be the same size - and the
#% size I've seen is 56 * 1????
#pp=[rsubp, csubp, i*ones(size(r,1),1)];
pp = np.c_[rsubp, csubp, i * np.ones((r.shape[0], 1))]
#% Alex: here we add more rows (pp) to points below the existing rows of
#% points
#points=[points; pp];
common.DebugPrint(
"multi_scale_harris_Evangelidis(): points.shape = %s" %
str(points.shape))
common.DebugPrint("multi_scale_harris_Evangelidis(): pp.shape = %s" %
str(pp.shape))
if points.size == 0:
# Avoiding exception: "ValueError: arrays must have same number of dimensions"
points = pp
else:
points = np.r_[points, pp]
#toc
if disp:
assert False
# not implemented the display of Harris features
"""
figure;
imshow(im,[]) #hold on
title('corners detected')
for i = range(1, size(points,1) + 1):
rectangle('Position',[points(i,2)-3*points(i,3),points(i,1)-3*points(i,3),...
2*3*points(i,3),2*3*points(i,3)],'Curvature',[1,1],'EdgeColor','w','LineWidth',2)
plot(points(i,2),points(i,1),'r+')
"""
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multi_scale_harris_Evangelidis(): points = %s" %
str(points))
common.DebugPrint("multi_scale_harris_Evangelidis(): points.shape = %s" %
str(points.shape))
tMSH2 = float(cv2.getTickCount())
myTime = (tMSH2 - tMSH1) / cv2.getTickFrequency()
print(
"multi_scale_harris_Evangelidis(): multi_scale_harris_Evangelidis() took %.6f [sec]"
% myTime)
return points
def multiscale_quad_retrieval(r_quadsTree, r_harlocs, q_harlocs, md_threshold, st_threshold, \
all_ori, all_id, all_max, all_cen, nos, scale_index, cropflag, \
sequence):
common.DebugPrint("Entered multiscale_quad_retrieval(): " \
"md_threshold = %s, st_threshold = %s." % \
(str(md_threshold), \
str(st_threshold)));
assert len(r_harlocs) != 0;
assert len(q_harlocs) != 0;
try:
Votes_space = np.load("Votes_space%d.npz" % scale_index)['arr_0'];
HH = np.load("HH%d.npz" % scale_index)['arr_0'];
return Votes_space, HH;
except:
common.DebugPrintErrorTrace();
if common.MY_DEBUG_STDOUT and DBGPRINT:
common.DebugPrint("multiscale_quad_retrieval(): r_quadsTree = %s" % \
str(r_quadsTree));
common.DebugPrint("multiscale_quad_retrieval(): len(r_harlocs) = %d" % len(r_harlocs));
common.DebugPrint("multiscale_quad_retrieval(): r_harlocs = %s" % str(r_harlocs));
common.DebugPrint("multiscale_quad_retrieval(): q_harlocs = %s" % str(q_harlocs));
common.DebugPrint("multiscale_quad_retrieval(): md_threshold = %s" % str(md_threshold));
print("multiscale_quad_retrieval(): st_threshold = %s" % str(st_threshold));
#common.DebugPrint("multiscale_quad_retrieval(): all_ori, all_id, all_max, all_cen, nos, scale_index, cropflag = %s" % str(all_ori, all_id, all_max, all_cen, nos, scale_index, cropflag));
common.DebugPrint("multiscale_quad_retrieval(): all_id = %s" % str(all_id));
common.DebugPrint("multiscale_quad_retrieval(): all_id.shape = %s" % (str(all_id.shape)));
#common.DebugPrint("multiscale_quad_retrieval(): all_max, all_cen, nos, scale_index, cropflag = %s" % str(all_max, all_cen, nos, scale_index, cropflag));
#common.DebugPrint("multiscale_quad_retrieval(): all_max = %s" % str(all_max));
#common.DebugPrint("multiscale_quad_retrieval(): all_cen, nos, scale_index, cropflag = %s" % str(all_cen, nos, scale_index, cropflag));
common.DebugPrint("multiscale_quad_retrieval(): sequence = %s" % str(sequence));
print("multiscale_quad_retrieval(): cropflag = %s" % str(cropflag));
t1 = float(cv2.getTickCount());
if scale_index > nos:
assert scale_index <= nos;
#error('Wrong scale index or number-of-scales');
#QD = dir([q_path "multiharlocs*.mat"])
#QD = [q_path + "multiharlocs*.mat"]
#QD = q_harlocs;
#RD = dir([r_path "multiharlocs*.mat"])
#RD = [r_path + "multiharlocs*.mat"]
#RD = r_harlocs;
#TODO: take out RD_start
#RD_start = str2num(RD(1).name(end - 9 : end - 4))
#RD_start = int(RD[0][-9 : -4])
RD_start = 0;
#RD_end = str2num(RD(end).name(end - 9 : end - 4))
#RD_end = int(RD[-1][-9 : -4])
#RD_end = len(RD) - 1;
RD_end = len(r_harlocs) - 1;
if False: # n_d not used anywhere
#n_d = hist(all_id, RD_start : RD_end)
#n_d = hist[all_id, RD_start : RD_end]
n_d = Matlab.hist(x=all_id, \
binCenters=np.array(range(RD_start, RD_end + 1)) );
#cross_indices = np.zeros( (len(QD), 2) );
cross_indices = np.zeros( (len(q_harlocs), 2) );
j = 1;
#tic
#ORI = np.array([]); # ORI NOT used anywhere
"""
Inspired from
https://stackoverflow.com/questions/17559140/matlab-twice-as-fast-as-numpy
BUT doesn't help in this case:
Votes_space = np.asfortranarray(np.zeros( (len(RD), len(QD)) ));
"""
#Votes_space = np.zeros( (len(RD), len(QD)) );
Votes_space = np.zeros( (len(r_harlocs), len(q_harlocs)) );
# Make a distinct copy of HH from Votes_space...
#HH = Votes_space.copy().astype(np.int16); #Votes_space + 0;
#HH = np.zeros((len(RD), len(QD)), dtype=np.int8);
HH = np.zeros((len(r_harlocs), len(q_harlocs)), dtype=np.int8); #!!!!TODO use MAYBE even np.bool - OR take it out
#common.DebugPrint("multiscale_quad_retrieval(): Votes_space = %s,\n HH = %s" % (str(Votes_space), str(HH)))
tolers = 0.1 - float(scale_index) / 100.0; # it helps to make more strict the threshold as the scale goes up
# tolers = 0.15 - float(scale_index) / 100.0;
MAXDIS = 3 + scale_index;
MAXORI = 0.25;
"""
!!!!TODO TODO: I am using multiprocessing.Poll and return votes;
the dispatcher assembles the results,
but the results are NOT the same with the serial case - although they
look pretty decent, but they seem to be suboptimal - dp_Alex returns
suboptimal cost path for USE_MULTITHREADING == True instead of
False.
(Note: running under the same preconditions
multiscale_quad_retrieval I got the same results in dp_Alex().
"""
if False: #config.USE_MULTITHREADING == True:
global g;
g.r_quadsTree = r_quadsTree;
g.r_harlocs = r_harlocs;
g.q_harlocs = q_harlocs;
g.md_threshold = md_threshold;
g.st_threshold = st_threshold;
g.all_ori = all_ori;
g.all_id = all_id;
g.all_max = all_max;
g.all_cen = all_cen;
g.nos = nos;
g.scale_index = scale_index;
g.cropflag = cropflag;
g.sequence = sequence;
g.RD_start = RD_start;
g.RD_end = RD_end;
g.MAXDIS = MAXDIS;
g.MAXORI = MAXORI;
g.tolers = tolers;
"""
Start worker processes to use on multi-core processor (able to run
in parallel - no GIL issue if each core has it's own VM)
"""
pool = multiprocessing.Pool(processes=config.numProcesses);
print("multiscale_quad_retrieval(): Spawned a pool of %d workers" % \
config.numProcesses);
listParams = range(0, len(q_harlocs)); #!!!!TODO: use counterStep, config.initFrame[indexVideo]
#res = pool.map(IterationStandaloneMQR, listParams);
# See https://docs.python.org/2/library/multiprocessing.html#module-multiprocessing.pool
res = pool.map(func=IterationStandaloneMQR, iterable=listParams, \
chunksize=1);
print("Pool.map returns %s" % str(res)); #x0.size + 1
"""
From https://medium.com/building-things-on-the-internet/40e9b2b36148
close the pool and wait for the work to finish
"""
pool.close();
pool.join();
# Doing the "reduce" phase after the workers have finished :)
assert len(res) == len(q_harlocs);
for queryFrame, resE in enumerate(res):
resEIndex = resE[0];
resE = resE[1];
assert resEIndex == queryFrame;
# Gives: "ValueError: output operand requires a reduction, but reduction is not enabled"
#Votes_space[:, queryFrame - 1] = votes;
Votes_space[:, queryFrame] = resE;
for queryFrame in range(len(q_harlocs)):
if cropflag == 0:
HH[:, queryFrame] = 1;
else:
"""
HH[:, queryFrame] = spatial_consistency.spatial_consistency(space_xy, \
qcen, len(RD), st_threshold, cropflag);
"""
HH[:, queryFrame] = spatial_consistency.spatial_consistency(space_xy, \
qcen, len(r_harlocs), st_threshold, cropflag);
try:
np.savez_compressed("Votes_space%d" % scale_index, Votes_space);
np.savez_compressed("HH%d" % scale_index, HH);
except:
common.DebugPrintErrorTrace();
return Votes_space, HH;
"""
We substitute q - 1 with q, since we want
to number arrays from 0 (not from 1 like in Matlab).
"""
#for q=1:length(QD)
#for q in range(1, len(QD) + 1):
#for queryFrame in range(len(QD)):
for queryFrame in range(len(q_harlocs)):
common.DebugPrint("multiscale_quad_retrieval(): Starting iteration queryFrame = %d" % queryFrame);
# tic
"""
str1=['load ' q_path QD(q).name]
eval(str1)
"""
"""
We make pp reference the desired multiharloc list for the query video
frame queryFrame
"""
pp = q_harlocs[queryFrame];
#pp = np.array(pp);
#common.DebugPrint("multiscale_quad_retrieval(): pp = %s" % str(pp));
#[qout,qcen,qmaxdis,qori]=findquads(pp(pp(:,3)==scale_index,1:2),md_threshold,0);
points = pp[pp[:, 2] == scale_index, 0:2];
qout, qcen, qmaxdis, qori = findquads.findquads(points, md_threshold, 0);
if common.MY_DEBUG_STDOUT and DBGPRINT:
print("multiscale_quad_retrieval(): queryFrame = %d, " \
"qout.shape (number of quads for query frame queryFrame) = %s" % \
(queryFrame, str(qout.shape)));
# disp([num2str(q) ' of ' num2str(length(QD)) ' -> ' num2str(size(qout,1)) ' quads'])
#space_xy=zeros(size(qcen,1),2*length(RD))+nan;
#space_xy = np.zeros( (qcen.shape[0], 2 * len(RD)) ) + np.nan;
space_xy = np.zeros( (qcen.shape[0], 2 * len(r_harlocs)) ) + np.nan;
# votes=zeros(length(RD),1)
#votes=zeros(length(RD),length(tolers));
#votes = np.zeros( (len(RD), 1) );
votes = np.zeros( (len(r_harlocs), 1) );
#nep = np.array([]);
#m_points = np.array([]);
assert isinstance(tolers, float);
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): quads of query frame %d are: " % queryFrame);
common.DebugPrint(" qout = %s" % str(qout));
"""
Alex: for each quad (4 floats) of the query frame from Harris feature of scale scale_index
Note: all_id stores the reference frame id for each quad descriptor.
"""
"""
We substitute queryFrameQuad - 1 with queryFrameQuad, since we want
to number arrays from 0 (not from 1 like in Matlab).
"""
#for queryFrameQuad in range(1, qout.shape[0] + 1):
for queryFrameQuad in range(qout.shape[0]):
common.DebugPrint("multiscale_quad_retrieval(): Starting iteration queryFrameQuad = %d" % queryFrameQuad);
"""
Matlab's polymorphism is really bugging here: although it's
normally a float, tolers is considered to be a size 1 vector...
so len(tolers) == 1
"""
#for tol_i in range(1, len(tolers) + 1):
# tol = tolers[tol_i - 1]
"""
We substitute tol_i - 1 with tol, since we want
to number arrays from 0 (not from 1 like in Matlab).
"""
#for tol_i in range(1, 1 + 1):
for tol_i in range(1):
tol = tolers;
"""
# TODO: done below - take out this dbg print
if DBGPRINT:
common.DebugPrint("multiscale_quad_retrieval(): " \
"qout[queryFrameQuad, :] = %s" % \
str(qout[queryFrameQuad, :]))
"""
#% default for first PAMI with tol= 0.1 approximately
# NOTE: SciPy's KDTree finds a few more results, in some cases,
# than the Matlab code from Evangelidis.
#idx, di = kdtree_ball_query(tree, qout(i, :), tol)
#idx, distKD = kdtree_ball_query(tree, qout[i - 1, :], tol)
#idx, di = tree.query(x=xQuery, k=4)
#resPoints = [data[i] for i in resBallIndices]
# tol is a scalar representing the radius of the ball
if config.KDTREE_IMPLEMENTATION == 0:
idx = r_quadsTree.query_ball_point(qout[queryFrameQuad, :], tol);
elif config.KDTREE_IMPLEMENTATION == 1:
#pt = qout[queryFrameQuad - 1, :].astype(np.float32);
pt = qout[queryFrameQuad, :];
pt = np.array([[pt[0], pt[1], pt[2], pt[3]]], dtype=np.float32);
retval, idx, dists = r_quadsTree.radiusSearch( \
query=pt, \
radius=(tol**2), \
maxResults=NUM_MAX_ELEMS, \
params=search_params);
if common.MY_DEBUG_STDOUT and DBGPRINT:
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"retval (number NNs) = %s" % str(retval));
"""
common.DebugPrint( \
"multiscale_quad_retrieval(): radiusSearch's retval " \
"(at queryFrame=%d, queryFrameQuad=%d) is %d" % (queryFrame, queryFrameQuad, retval));
idx = idx[0];
dists = dists[0];
"""
Note: retval is the number of neighbors returned from the radiusSearch().
But the idx and the dists can have more elements than the returned retval.
"""
idx = idx[: retval];
dists = dists[: retval];
if common.MY_DEBUG_STDOUT and DBGPRINT:
print("multiscale_quad_retrieval(): " \
"qout[queryFrameQuad, :] = %s" % str(qout[queryFrameQuad, :]));
print("multiscale_quad_retrieval(): " \
"idx = %s" % str(idx));
print("multiscale_quad_retrieval(): " \
"dists = %s" % str(dists));
print("multiscale_quad_retrieval(): " \
"tol = %s" % str(tol));
if config.KDTREE_IMPLEMENTATION == 0:
print("multiscale_quad_retrieval(): " \
"r_quadsTree.data[idx] = %s" % \
str(r_quadsTree.data[idx]));
# We print the distances to the points returned in idx
if common.MY_DEBUG_STDOUT and DBGPRINT: # This is just for debugging purposes
a = qout[queryFrameQuad, :];
if config.KDTREE_IMPLEMENTATION == 0:
for myI, index in enumerate(idx):
b = r_quadsTree.data[index];
"""
if False:
common.DebugPrint("multiscale_quad_retrieval(): distance to " \
"%d point (%s) inside ball = %.4f" % \
(myI, str(b), npla.norm(a - b)));
"""
else:
pass;
idx = np.array(idx);
#if False:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): " \
"all_max.shape = %s" % str(all_max.shape));
common.DebugPrint("multiscale_quad_retrieval(): " \
"qmaxdis.shape = %s" % str(qmaxdis.shape));
common.DebugPrint("multiscale_quad_retrieval(): " \
"qmaxdis = %s" % str(qmaxdis));
common.DebugPrint("multiscale_quad_retrieval(): " \
"qori.shape = %s" % str(qori.shape));
common.DebugPrint("multiscale_quad_retrieval(): " \
"qori = %s" % str(qori));
#dis_idx=abs(qmaxdis(i)-all_max(idx))<MAXDIS;
if len(idx) == 0:
# NOT A GOOD IDEA: continue;
#idx = np.array([]);
dis_idx = np.array([]);
ori_idx = np.array([]);
else:
if common.MY_DEBUG_STDOUT and DBGPRINT:
print("multiscale_quad_retrieval(): " \
"queryFrameQuad = %s" % str(queryFrameQuad));
print("multiscale_quad_retrieval(): " \
"all_max[idx] = %s" % str(all_max[idx]));
print("multiscale_quad_retrieval(): " \
"qmaxdis[queryFrameQuad] = %s" % str(qmaxdis[queryFrameQuad]));
if USE_GPS_COORDINATES:
# We look only at a part of the reference video
"""
Since in some cases the video temporal alignment is
difficult to do due to similar portions in the
trajectory (see the drone videos, clip 3_some_lake)
we "guide" the temporal alignment by restricting
the reference frame search space - this is useful
when we have the geolocation (GPS) coordinate for
each frame.
"""
if common.MY_DEBUG_STDOUT and DBGPRINT:
print("multiscale_quad_retrieval(): " \
"all_id = %s" % str(all_id));
if True:
#assert (all_id.ndim == 2) and (all_id.shape[1] == 1);
if all_id.ndim == 2:
#!!!!TODO TODO: put this at the beginning of the function
assert all_id.shape[1] == 1;
"""
We flatten the array all_id
Note: We don't use order="F" since it's
basically 1-D array
"""
all_id = np.ravel(all_id);
#!!!!TODO: put start and end frame in config - or compute it from geolocation
sub_idx = np.logical_and( (all_id[idx] >= 2030 - 928), \
(all_id[idx] <= 2400 - 928) );
idx = idx[sub_idx];
if common.MY_DEBUG_STDOUT and DBGPRINT:
print("multiscale_quad_retrieval(): " \
"all_id = %s" % str(all_id));
print("multiscale_quad_retrieval(): " \
"sub_idx = %s" % str(sub_idx));
print("multiscale_quad_retrieval(): " \
"idx = %s" % str(idx));
if FILTER:
dis_idx = np.abs(qmaxdis[queryFrameQuad] - all_max[idx]) < MAXDIS;
#if False:
if common.MY_DEBUG_STDOUT:
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"idx = %s" % str(idx));
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"dis_idx = %s" % str(dis_idx));
#idx=idx(dis_idx)
idx = idx[dis_idx];
#if False:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): " \
"idx (after idx = idx[dis_idx]) = %s" % str(idx));
if FILTER:
#ori_idx=abs(qori(i)-all_ori(idx))<MAXORI;
ori_idx = np.abs(qori[queryFrameQuad] - all_ori[idx]) < MAXORI;
#if False:
if common.MY_DEBUG_STDOUT:
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"all_ori = %s" % str(all_ori));
common.DebugPrint("multiscale_quad_retrieval(): " \
"qori[queryFrameQuad] = %s" % str(qori[queryFrameQuad]));
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"ori_idx = %s" % str(ori_idx));
#idx=idx(ori_idx);
idx = idx[ori_idx];
# IMPORTANT ###################################################
# IMPORTANT ###################################################
# IMPORTANT ###################################################
#% spatio-temporal consistency
# IMPORTANT ###################################################
# IMPORTANT ###################################################
# IMPORTANT ###################################################
#if numel(idx) > 0:
if idx.size > 0:
if cropflag == 0:
if FILTER:
"""
Alex: this is a simple procedure of eliminating False
Positive (FP) matches, as presented in Section 4.2 of
TPAMI 2013 paper.
Basically it filters out quad matches that have
centroids st_threshold away from the query quad.
Note: all_cen are the controids of all reference
quads.
"""
dy = qcen[queryFrameQuad, 0] - all_cen[idx, 0];
dx = qcen[queryFrameQuad, 1] - all_cen[idx, 1];
#D=dy.^2+dx.^2;
D = dy**2 + dx**2;
co_idx = D < pow(st_threshold, 2);
idx = idx[co_idx];
else:
"""
We substitute iii - 1 with iii, since we want
to number arrays from 0 (not from 1 like in Matlab).
"""
#for iii in range(1, len(idx) + 1):
for iii in range(len(idx)):
#space_xy(i,(all_id(idx(iii))-RD_start)*2+1:(all_id(idx(iii))-RD_start)*2+2) = all_cen(idx(iii),:)
space_xy[queryFrameQuad, \
(all_id[idx[iii]] - RD_start) * 2: (all_id[idx[iii] - 1] - RD_start) * 2 + 1] = \
all_cen[idx[iii], :];
#hh=hist(all_id(idx),RD_start:RD_end);
# It has to be an np.array because we multiply it with a scalar
histoRange = np.array(range(RD_start, RD_end + 1));
hh = Matlab.hist(x=all_id[idx], binCenters=histoRange);
#if False:
#if True:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): " \
"hh = %s" % (str(hh)));
common.DebugPrint("multiscale_quad_retrieval(): " \
"hh.shape = %s" % (str(hh.shape)));
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"all_id = %s" % (str(all_id)));
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"all_id.shape = %s" % (str(all_id.shape)));
common.DebugPrint("multiscale_quad_retrieval(): " \
"idx = %s" % (str(idx)));
common.DebugPrint("multiscale_quad_retrieval(): " \
"idx.shape = %s" % (str(idx.shape)));
# % nz can be computed more optimally
#nz=find(hh~=0); # nz can be computed more optimally
# np.nonzero() always returns a tuple, even if it contains 1 element since hh has only 1 dimension
nz = np.nonzero(hh != 0)[0];
#if False:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): " \
"nz = %s" % (str(nz)));
common.DebugPrint("multiscale_quad_retrieval(): " \
"nz.shape = %s" % (str(nz.shape)));
#if numel(nz) > 0
if nz.size > 0:
#%%----text-retrieval-like
#votes(nz, tol_i) = votes(nz, tol_i) + log10(length(RD) / (length(nz)))^2 #PREVIOUSLY
#myVal = pow(math.log10(float(len(RD)) / len(nz)), 2);
myVal = pow(math.log10(float(len(r_harlocs)) / len(nz)), 2);
"""
try:
myVal = pow(math.log10(float(len(r_harlocs)) / len(nz)), 2);
except:
print("Error: len=%d len(nz)=%d nz.size=%d" % \
(len(r_harlocs), len(nz), nz.size));
common.DebugPrintErrorTrace();
"""
#if False:
if common.MY_DEBUG_STDOUT:
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"len(RD) = %d" % len(RD));
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"len(r_harlocs) = %d" % len(r_harlocs));
common.DebugPrint("multiscale_quad_retrieval(): " \
"len(nz) = %d" % len(nz));
common.DebugPrint("multiscale_quad_retrieval(): " \
"myVal = %.5f" % myVal);
# PREVIOUSLY
votes[nz, tol_i] = votes[nz, tol_i] + myVal;
# votes(nz)=votes(nz)+log10(length(RD)/(length(nz)));
# votes(nz)=votes(nz)+1;
if common.MY_DEBUG_STDOUT and DBGPRINT:
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"Votes_space.shape = %s" % (str(Votes_space.shape)));
common.DebugPrint("multiscale_quad_retrieval(): " \
"votes.shape = %s" % (str(votes.shape)));
"""
print("multiscale_quad_retrieval(): " \
"votes.shape = %s" % (str(votes.shape)));
if (np.abs(votes) < 1.0e-10).all():
print( \
"multiscale_quad_retrieval(): votes = 0 (all zeros)");
else:
print("multiscale_quad_retrieval(): " \
"votes = %s" % (str(votes)));
#Votes_space(:,q)=votes;
# Gives: "ValueError: output operand requires a reduction, but reduction is not enabled"
#Votes_space[:, queryFrame - 1] = votes;
# Note: since votes is basically a 1-D vector, we don't use the Fortran order
Votes_space[:, queryFrame] = np.ravel(votes); # order="F");
if cropflag == 0:
HH[:, queryFrame] = 1;
else:
"""
HH[:, queryFrame] = spatial_consistency.spatial_consistency(space_xy, \
qcen, len(RD), st_threshold, cropflag);
"""
HH[:, queryFrame] = spatial_consistency.spatial_consistency(space_xy, \
qcen, len(r_harlocs), st_threshold, cropflag);
if common.MY_DEBUG_STDOUT and DBGPRINT:
print("multiscale_quad_retrieval(scale_index=%d): " \
"Votes_space =\n%s" % (scale_index, str(Votes_space)));
try:
np.savez_compressed("Votes_space%d" % scale_index, Votes_space);
np.savez_compressed("HH%d" % scale_index, HH);
except:
common.DebugPrintErrorTrace();
t2 = float(cv2.getTickCount());
myTime = (t2 - t1) / cv2.getTickFrequency();
print("multiscale_quad_retrieval() took %.6f [sec]" % myTime);
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"%d corresponding frames retrieved in %.6f secs" % \
(len(q_harlocs), myTime));
"""
return Votes_space, HH;
def ComputeHarlocs(capture,
counterStep,
folderName,
fileNamePrefix,
fileNameExtension=".csv",
indexVideo=-1):
print( \
"Entered ComputeHarlocs(capture=%s, counterStep=%d, folderName=%s, " \
"indexVideo=%d)" % \
(str(capture), counterStep, folderName, indexVideo))
harlocsFolder = config.VIDEOS_FOLDER + "/" + folderName
t1 = float(cv2.getTickCount())
harlocs = []
if config.OCV_OLD_PY_BINDINGS:
numFrames = int(capture.get(cv2.cv.CV_CAP_PROP_FRAME_COUNT))
else:
numFrames = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))
common.DebugPrint("ComputeHarlocs(): numFrames = %d" % numFrames)
if not os.path.exists(harlocsFolder):
os.makedirs(harlocsFolder)
else:
#!!!!TODO: check that the loaded Harlocs are complete - same frame numbers as in the videos
# Folder with precomputed Harris features exists
folderContent = os.listdir(harlocsFolder)
sortedFolderContent = sorted(folderContent)
for fileName in sortedFolderContent:
pathFileName = harlocsFolder + "/" + fileName
"""
common.DebugPrint("ComputeHarlocs(): pathFileName = %s" % pathFileName);
common.DebugPrint("ComputeHarlocs(): fileName = %s" % fileName);
"""
if os.path.isfile(pathFileName) and \
fileName.startswith(fileNamePrefix) and \
pathFileName.endswith(fileNameExtension):
common.DebugPrint("ComputeHarlocs(): Loading %s" %
pathFileName)
harrisFeatures = multi_scale_harris.LoadMultiScaleHarrisFeatures(
pathFileName)
harlocs.append(harrisFeatures)
if config.endFrame[indexVideo] == -1:
assert (len(harlocs) + config.initFrame[indexVideo]) == numFrames
#!!!!TODO: if condition is NOT met, give a nicer error, or redo computations of Harlocs
else:
assert (len(harlocs) + config.initFrame[indexVideo]
) == config.endFrame[indexVideo] + 1
#!!!!TODO: if condition is NOT met, give a nicer error, or redo computations of Harlocs
return harlocs
if config.USE_MULTITHREADING == True:
global g
g.captureQ = None
# We need to reopen the capture device in each process, separately
g.captureR = None
# We need to reopen the capture device in each process, separately
#g.capture = capture;
g.harlocsFolder = harlocsFolder
g.fileNamePrefix = fileNamePrefix
g.fileNameExtension = fileNameExtension
g.indexVideo = indexVideo
if config.OCV_OLD_PY_BINDINGS:
frameCount = int(capture.get(cv2.cv.CV_CAP_PROP_FRAME_COUNT))
else:
frameCount = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))
#listParams = range(frameCount);
listParams = range(config.initFrame[indexVideo], frameCount,
counterStep)
common.DebugPrint("ComputeHarlocs(): frameCount = %d" % frameCount)
print("ComputeHarlocs(): Spawning a pool of %d workers" % \
config.numProcesses)
"""
# DEBUG purposes ONLY - since when we use Pool() and call function, if
# we have an error in the function the exception reported is very
# vague...
for i in listParams:
IterationStandaloneMSH(i);
#import time
#time.sleep(1000);
"""
"""
Start worker processes to use on multi-core processor (circumvent
also the GIL issue).
"""
pool = multiprocessing.Pool(processes=config.numProcesses)
print("ComputeHarlocs(): Spawned a pool of %d workers" % \
config.numProcesses)
#res = pool.map(IterationStandaloneMSH, listParams);
# See https://docs.python.org/2/library/multiprocessing.html#module-multiprocessing.pool
res = pool.map(func=IterationStandaloneMSH, iterable=listParams, \
chunksize=1)
print("Pool.map returns %s" % str(res))
"""
From https://medium.com/building-things-on-the-internet/40e9b2b36148
close the pool and wait for the work to finish
"""
pool.close()
pool.join()
#!!!!TODO: do more efficient - don't load the results from the CSV files
return ComputeHarlocs(capture, counterStep, folderName, \
fileNamePrefix, fileNameExtension, indexVideo)
#return [];
#indexHarloc = 0;
while True:
if config.OCV_OLD_PY_BINDINGS:
framePos = capture.get(cv2.cv.CV_CAP_PROP_POS_FRAMES)
else:
"""
From http://docs.opencv.org/modules/highgui/doc/reading_and_writing_images_and_video.html#videocapture-get:
<<CV_CAP_PROP_POS_FRAMES 0-based index of the frame to be decoded/captured next.>>
"""
framePos = capture.get(cv2.CAP_PROP_POS_FRAMES)
common.DebugPrint("ComputeHarlocs(): framePos = %d" % framePos)
counter = int(framePos)
#0
common.DebugPrint("ComputeHarlocs(): counter = %d" % counter)
ret, img = capture.read()
if common.MY_DEBUG_STDOUT:
common.DebugPrint("ComputeHarlocs(): img = %s" % str(img))
if False and config.SAVE_FRAMES:
fileName = config.IMAGES_FOLDER + "/img_%05d.png" % counter
if not os.path.exists(fileName):
#print "dir(img) = %s"% str(dir(img))
"""
imgCV = cv.fromarray(img)
cv2.imwrite(fileName, imgCV)
"""
cv2.imwrite(fileName, img)
#if ret == False: #MatchFrames.counterQ == 3:
if (ret == False) or ((counter > numFrames) or \
(config.endFrame[indexVideo] != -1 and \
counter > config.endFrame[indexVideo])):
break
if config.VIDEO_FRAME_RESIZE_SCALING_FACTOR != 1:
img = Matlab.imresize(img, \
scale=config.VIDEO_FRAME_RESIZE_SCALING_FACTOR)
im = img
pp = multi_scale_harris.multi_scale_harris(im, nos, disp=0)
# n=0:nos-1
#harlocs = pp
harlocs.append(pp)
multi_scale_harris.StoreMultiScaleHarrisFeatures( \
harlocsFolder + "/" + fileNamePrefix + "%05d%s" % \
(counter, fileNameExtension),
pp)
counter += counterStep
# If we try to seek to a frame out-of-bounds frame it gets to the last one
if config.OCV_OLD_PY_BINDINGS:
capture.set(cv2.cv.CV_CAP_PROP_POS_FRAMES, counter)
else:
capture.set(cv2.CAP_PROP_POS_FRAMES, counter)
#indexHarloc += 1;
t2 = float(cv2.getTickCount())
myTime = (t2 - t1) / cv2.getTickFrequency()
common.DebugPrint("ComputeHarlocs(): computing the multiscale harlocs " \
"took %.6f [sec]" % myTime)
#common.DebugPrint("ComputeHarlocs(): len(harlocs) = %s" % str(len(harlocs)));
if False:
for i, h in enumerate(harlocs):
#common.DebugPrint("ComputeHarlocs(): len(harlocs[%d]) = %d" % \
# (i, len(h)));
multi_scale_harris.StoreMultiScaleHarrisFeatures(
harlocsFolder + "/" + fileNamePrefix + "%05d.txt" % i, h)
#if False:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("ComputeHarlocs(): harlocs = %s" % str(harlocs))
return harlocs
def IterationStandaloneMQR(queryFrame):
r_quadsTree = g.r_quadsTree;
r_harlocs = g.r_harlocs;
q_harlocs = g.q_harlocs;
md_threshold = g.md_threshold;
st_threshold = g.st_threshold;
all_ori = g.all_ori;
all_id = g.all_id;
all_max = g.all_max;
all_cen = g.all_cen;
nos = g.nos;
scale_index = g.scale_index;
cropflag = g.cropflag;
sequence = g.sequence;
RD_start = g.RD_start;
RD_end = g.RD_end;
MAXDIS = g.MAXDIS;
MAXORI = g.MAXORI;
tolers = g.tolers;
"""
common.DebugPrint( \
"Entered IterationStandaloneMQR(): crossref=%s, captureQ=%s, "\
"captureR=%s, refined_crossref=%s, warp_p=%s, "
"x0=%s, y0=%s, start=%s, t=%d, iWhile=%d." % \
(str(crossref), str(captureQ), str(captureR), \
str(g.refined_crossref), str(g.warp_p), \
str(g.x0), str(g.y0), str(g.start), g.t, iWhile));
common.DebugPrint("IterationStandalone(): id(g)=%s" % str(id(g)));
"""
# tic
"""
str1=['load ' q_path QD(q).name]
eval(str1)
"""
"""
We make pp reference the desired multiharloc list for the query video
frame queryFrame
"""
pp = q_harlocs[queryFrame];
#pp = np.array(pp);
#common.DebugPrint("multiscale_quad_retrieval(): pp = %s" % str(pp));
"""
Alex: for the query frame queryFrame we retrieve, for scale scale_index, the
harris features in var points.
Then we build the quads from points.
Then for each quad (4 float values) we query the corresponding scale
kd-tree, and we get the indices.
Then we build the histogram and compute idf, ....!!!!
Note: scale is 1 for original frame resolution and the higher
we go we have lower image resolutions (we go higher in the
Guassian pyramid I think).
"""
#[qout,qcen,qmaxdis,qori]=findquads(pp(pp(:,3)==scale_index,1:2),md_threshold,0);
points = pp[pp[:, 2] == scale_index, 0:2];
qout, qcen, qmaxdis, qori = findquads.findquads(points, md_threshold, 0);
common.DebugPrint("multiscale_quad_retrieval(): queryFrame = %d, " \
"qout.shape = %s" % (queryFrame, str(qout.shape)));
# disp([num2str(q) ' of ' num2str(length(QD)) ' -> ' num2str(size(qout,1)) ' quads'])
#space_xy=zeros(size(qcen,1),2*length(RD))+nan;
#space_xy = np.zeros( (qcen.shape[0], 2 * len(RD)) ) + np.nan;
space_xy = np.zeros( (qcen.shape[0], 2 * len(r_harlocs)) ) + np.nan;
# votes=zeros(length(RD),1)
#votes=zeros(length(RD),length(tolers));
#votes = np.zeros( (len(RD), 1) );
votes = np.zeros( (len(r_harlocs), 1) );
#nep = np.array([]);
#m_points = np.array([]);
assert isinstance(tolers, float);
"""
We substitute queryFrameQuad - 1 with queryFrameQuad, since we want
to number arrays from 0 (not from 1 like in Matlab).
"""
#for queryFrameQuad in range(1, qout.shape[0] + 1):
for queryFrameQuad in range(qout.shape[0]):
"""
Matlab's polymorphism is really bugging here: although it's
normally a float, tolers is considered to be a size 1 vector...
so len(tolers) == 1
"""
#for tol_i in range(1, len(tolers) + 1):
# tol = tolers[tol_i - 1]
"""
We substitute tol_i - 1 with tol, since we want
to number arrays from 0 (not from 1 like in Matlab).
"""
#for tol_i in range(1, 1 + 1):
for tol_i in range(1):
tol = tolers;
#common.DebugPrint("multiscale_quad_retrieval(): qout[i - 1, :] = %s" % str(qout[i - 1, :]))
#% default for first PAMI with tol= 0.1 approximately
# NOTE: SciPy's KDTree finds a few more results, in some cases,
# than the Matlab code from Evangelidis.
#idx, di = kdtree_ball_query(tree, qout(i, :), tol)
#idx, distKD = kdtree_ball_query(tree, qout[i - 1, :], tol)
#idx, di = tree.query(x=xQuery, k=4)
#resPoints = [data[i] for i in resBallIndices]
# tol is a scalar representing the radius of the ball
if config.KDTREE_IMPLEMENTATION == 0:
idx = r_quadsTree.query_ball_point(qout[queryFrameQuad, :], tol);
elif config.KDTREE_IMPLEMENTATION == 1:
#pt = qout[queryFrameQuad - 1, :].astype(np.float32);
pt = qout[queryFrameQuad, :];
pt = np.array([[pt[0], pt[1], pt[2], pt[3]]], dtype=np.float32);
retval, idx, dists = r_quadsTree.radiusSearch( \
query=pt, \
radius=(tol**2), \
maxResults=NUM_MAX_ELEMS, \
params=search_params);
if common.MY_DEBUG_STDOUT and DBGPRINT:
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"retval (number NNs) = %s" % str(retval));
"""
common.DebugPrint( \
"multiscale_quad_retrieval(): radiusSearch's retval " \
"(at queryFrame=%d, queryFrameQuad=%d) is %d\n" % (queryFrame, queryFrameQuad, retval));
idx = idx[0];
dists = dists[0];
idx = idx[: retval];
dists = dists[: retval];
if common.MY_DEBUG_STDOUT and DBGPRINT:
print("multiscale_quad_retrieval(): " \
"qout[queryFrameQuad, :] = %s" % str(qout[queryFrameQuad, :]));
print("multiscale_quad_retrieval(): " \
"idx = %s" % str(idx));
print("multiscale_quad_retrieval(): " \
"tol = %s" % str(tol));
if config.KDTREE_IMPLEMENTATION == 0:
print("multiscale_quad_retrieval(): " \
"r_quadsTree.data[idx] = %s" % \
str(r_quadsTree.data[idx]));
# We print the distances to the points returned in idx
a = qout[queryFrameQuad, :];
if False: #!!!! This is just for debugging purposes
for myI, index in enumerate(idx):
b = r_quadsTree.data[index];
"""
if False:
common.DebugPrint("multiscale_quad_retrieval(): distance to " \
"%d point (%s) inside ball = %.4f" % \
(myI, str(b), npla.norm(a - b)));
"""
idx = np.array(idx);
#if False:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): " \
"all_max.shape = %s" % str(all_max.shape));
common.DebugPrint("multiscale_quad_retrieval(): " \
"qmaxdis.shape = %s" % str(qmaxdis.shape));
common.DebugPrint("multiscale_quad_retrieval(): " \
"qmaxdis = %s" % str(qmaxdis));
common.DebugPrint("multiscale_quad_retrieval(): " \
"qori.shape = %s" % str(qori.shape));
common.DebugPrint("multiscale_quad_retrieval(): " \
"qori = %s" % str(qori));
#dis_idx=abs(qmaxdis(i)-all_max(idx))<MAXDIS;
if len(idx) == 0:
# NOT A GOOD IDEA: continue;
#idx = np.array([]);
dis_idx = np.array([]);
ori_idx = np.array([]);
else:
#if False:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): " \
"queryFrameQuad = %s" % str(queryFrameQuad));
common.DebugPrint("multiscale_quad_retrieval(): " \
"all_max[idx] = %s" % str(all_max[idx]));
common.DebugPrint("multiscale_quad_retrieval(): " \
"qmaxdis[queryFrameQuad] = %s" % str(qmaxdis[queryFrameQuad]));
dis_idx = np.abs(qmaxdis[queryFrameQuad] - all_max[idx]) < MAXDIS;
#if False:
if common.MY_DEBUG_STDOUT:
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"idx = %s" % str(idx));
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"dis_idx = %s" % str(dis_idx));
#idx=idx(dis_idx)
idx = idx[dis_idx];
#if False:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): " \
"idx (after idx = idx[dis_idx]) = %s" % str(idx));
#ori_idx=abs(qori(i)-all_ori(idx))<MAXORI;
ori_idx = np.abs(qori[queryFrameQuad] - all_ori[idx]) < MAXORI;
#if False:
if common.MY_DEBUG_STDOUT:
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"all_ori = %s" % str(all_ori));
common.DebugPrint("multiscale_quad_retrieval(): " \
"qori[queryFrameQuad] = %s" % str(qori[queryFrameQuad]));
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"ori_idx = %s" % str(ori_idx));
#idx=idx(ori_idx);
idx = idx[ori_idx];
# IMPORTANT ###################################################
# IMPORTANT ###################################################
# IMPORTANT ###################################################
#% spatio-temporal consistency
# IMPORTANT ###################################################
# IMPORTANT ###################################################
# IMPORTANT ###################################################
#if numel(idx) > 0:
if idx.size > 0:
# Normally cropflag == 0
if cropflag == 0:
dy = qcen[queryFrameQuad, 0] - all_cen[idx, 0];
dx = qcen[queryFrameQuad, 1] - all_cen[idx, 1];
#D=dy.^2+dx.^2;
D = dy**2 + dx**2;
co_idx = D < pow(st_threshold, 2);
idx = idx[co_idx];
else:
"""
We substitute iii - 1 with iii, since we want
to number arrays from 0 (not from 1 like in Matlab).
"""
#for iii in range(1, len(idx) + 1):
for iii in range(len(idx)):
#space_xy(i,(all_id(idx(iii))-RD_start)*2+1:(all_id(idx(iii))-RD_start)*2+2) = all_cen(idx(iii),:)
space_xy[queryFrameQuad, \
(all_id[idx[iii]] - RD_start) * 2: (all_id[idx[iii] - 1] - RD_start) * 2 + 1] = \
all_cen[idx[iii], :]
#hh=hist(all_id(idx),RD_start:RD_end);
# It has to be an np.array because we multiply it with a scalar
histoRange = np.array(range(RD_start, RD_end + 1));
hh = Matlab.hist(x=all_id[idx], binCenters=histoRange);
#if False:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): " \
"hh = %s" % (str(hh)));
common.DebugPrint("multiscale_quad_retrieval(): " \
"hh.shape = %s" % (str(hh.shape)));
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"all_id = %s" % (str(all_id)));
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"all_id.shape = %s" % (str(all_id.shape)));
common.DebugPrint("multiscale_quad_retrieval(): " \
"idx = %s" % (str(idx)));
common.DebugPrint("multiscale_quad_retrieval(): " \
"idx.shape = %s" % (str(idx.shape)));
# % nz can be computed more optimally
#nz=find(hh~=0); # nz can be computed more optimally
# np.nonzero() always returns a tuple, even if it contains 1 element since hh has only 1 dimension
nz = np.nonzero(hh != 0)[0];
#if False:
if common.MY_DEBUG_STDOUT:
common.DebugPrint("multiscale_quad_retrieval(): " \
"nz = %s" % (str(nz)));
common.DebugPrint("multiscale_quad_retrieval(): " \
"nz.shape = %s" % (str(nz.shape)));
#if numel(nz) > 0:
if nz.size > 0:
#%%----text-retrieval-like
#votes(nz, tol_i) = votes(nz, tol_i) + log10(length(RD) / (length(nz)))^2 #Note: log10(a)^2 means (log10(a))^2 #PREVIOUSLY
#myVal = pow(math.log10(float(len(RD)) / len(nz)), 2);
myVal = pow(math.log10(float(len(r_harlocs)) / len(nz)), 2);
#if False:
if common.MY_DEBUG_STDOUT:
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"len(RD) = %d" % len(RD));
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"len(r_harlocs) = %d" % len(r_harlocs));
common.DebugPrint("multiscale_quad_retrieval(): " \
"len(nz) = %d" % len(nz));
common.DebugPrint("multiscale_quad_retrieval(): " \
"myVal = %.5f" % myVal);
# PREVIOUSLY
votes[nz, tol_i] = votes[nz, tol_i] + myVal;
# votes(nz)=votes(nz)+log10(length(RD)/(length(nz)));
# votes(nz)=votes(nz)+1;
#if False:
if common.MY_DEBUG_STDOUT:
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"Votes_space.shape = %s" % (str(Votes_space.shape)));
common.DebugPrint("multiscale_quad_retrieval(): " \
"votes.shape = %s" % (str(votes.shape)));
"""
common.DebugPrint("multiscale_quad_retrieval(): " \
"votes.shape = %s" % (str(votes.shape)));
common.DebugPrint("multiscale_quad_retrieval(): " \
"votes = %s" % (str(votes)));
return (queryFrame, np.ravel(votes));
# NOT performing these in each worker - the central dispatcher will do these
if False:
#Votes_space(:,q)=votes;
# Gives: "ValueError: output operand requires a reduction, but reduction is not enabled"
#Votes_space[:, queryFrame - 1] = votes;
Votes_space[:, queryFrame] = np.ravel(votes);
if cropflag == 0:
HH[:, queryFrame] = 1;
else:
"""
HH[:, queryFrame] = spatial_consistency.spatial_consistency(space_xy, \
qcen, len(RD), st_threshold, cropflag);
"""
HH[:, queryFrame] = spatial_consistency.spatial_consistency(space_xy, \
qcen, len(r_harlocs), st_threshold, cropflag);
def calculate_trigger(self):
pt_cnt = self.point_count.currentIndex()
if (pt_cnt == 0):
raise Exception("Point count cannot be 0")
self.silo.getSilo().set_point_count(pt_cnt)
self.points_widget.update_silo(self.silo)
repose = self.textbox_repose_angle.text()
if (repose == ""):
raise Exception("Must give repose angle")
repose = float(repose)
self.silo.getSilo().set_repose_angle(repose)
error = self.textbox_percent_error.text()
if (error == ""):
raise Exception("Must give percent error")
error = float(error)
self.silo.getSilo().set_percent_error(error)
# perform calculation
fill_pts = []
draw_pts = []
for entry in self.silo.getSilo().get_point_pos():
if entry.isFill():
fill_pts.append([entry.getY(), entry.getX()])
else:
draw_pts.append([entry.getX(), entry.getX()])
if isinstance(self.silo, Silo.SiloCircle):
pile_struct = Matlab.pile(
fill_pts, draw_pts,
[self.silo.getRadius() * 2,
self.silo.getRadius() * 2], repose)
else:
pile_struct = Matlab.pile(
fill_pts, draw_pts,
[self.silo.getSideLen(),
self.silo.getSideLen()], repose)
error_struct = Matlab.error(pile_struct["x"], pile_struct["y"],
pile_struct["z"])
self.silo.getSilo().set_plot(pile_struct["x"], pile_struct["y"],
pile_struct["z"])
self.graph.graph3d.set_mesh(pile_struct["x"], pile_struct["y"],
pile_struct["z"])
self.graph.graph3d.canvas.draw()
red_x = []
red_y = []
red_z = []
for row_i in range(len(error_struct["data"])):
for col_i in range(len(error_struct["data"][row_i])):
if (error_struct["map"][row_i, col_i] == 1):
red_x.append(pile_struct["x"][row_i, col_i])
red_y.append(pile_struct["y"][row_i, col_i])
red_z.append(error_struct["data"][row_i, col_i])
self.silo.getSilo().set_min_plot(red_x, red_y, red_z)
self.graph.graph3d_red.set_mesh(red_x, red_y, red_z)
self.graph.graph3d_red.canvas.draw()
self.silo.getSilo().set_sensor_locale(red_x, red_y)
self.graph.graph_pos.set_mesh(red_x, red_y)
self.graph.graph_pos.canvas.draw()
error_index = []
for i in range(len(error_struct["error_list"])):
error_index.append(i + error_struct["start_count"])
error_index.reverse()
self.silo.getSilo().set_error_plot(error_index,
error_struct["error_list"])
self.graph.graph_error.set_mesh(error_index,
error_struct["error_list"])
self.graph.graph_error.canvas.draw()
pass
def my_nms(cim, radius, thresh):
common.DebugPrint("Entered my_nms(cim.shape=%s, radius=%s, thresh=%s)" % \
(str(cim.shape), str(radius), str(thresh)));
#%% modification of Peter-Kovesi non-maximum suppression software by
#%% G.Evangelidis
common.DebugPrint("my_nms(): cim = %s" % str(cim));
#%subPixel = nargout == 4; % We want sub-pixel locations
#subPixel=1;
subPixel = 1;
#[rows,cols] = size(cim)
rows, cols = cim.shape;
common.DebugPrint("my_nms(): rows, cols (cim.shape) = %s" % str((rows, cols)));
#% Extract local maxima by performing a grey scale morphological
#% dilation and then finding points in the corner strength image that
#% match the dilated image and are also greater than the threshold.
sze = 2 * radius + 1 #% Size of dilation mask.
common.DebugPrint("my_nms(): cim.shape = %s" % str(cim.shape));
#common.DebugPrint("my_nms(): cim = %s" % str(cim));
common.DebugPrint("my_nms(): sze = %s" % str(sze));
"""
Alex: we pass sze only being odd number and order = sze^2 makes ordfilt2()
return the maximum from the domain.
"""
#%% This modification runs 4x faster (11 secs less in a 6-scale approach)
#mx = ordfilt2(cim,sze^2,ones(sze)); #% Grey-scale dilate.
mx = Matlab.ordfilt2(cim, pow(sze, 2), np.ones((sze, sze))); #% Grey-scale dilate.
if common.MY_DEBUG_STDOUT:
common.DebugPrint("my_nms(): mx = %s" % str(mx));
#% mx=my_max_filter(cim,[sze,sze]); %my mex-file for max-filter
#%% This modification verify that the central point is the unique maximum in
#%% neighborhood
#% mx2= ordfilt2(cim,sze^2-1,ones(sze)); % This is used to vrify that the
#% central point is the unique maximum in neighborhood
#% imagesc(cim)
#% pause
#% imagesc(mx)
#% pause
#% close
#% Make mask to exclude points within radius of the image boundary.
#bordermask = zeros(size(cim));
#bordermask = np.zeros(cim.shape);
bordermask = np.zeros(cim.shape, dtype=np.uint8);
#Alex: sets to 1 the matrix, except the first and last radius lines and first and last radius columns, which are left 0
#bordermask(radius+1:end-radius, radius+1:end-radius) = 1;
bordermask[radius: -radius, radius: -radius] = 1;
#% Find maxima, threshold, and apply bordermask
#cimmx = (cim==mx) & (cim>thresh) & bordermask; #%Peter-Kovesi
cimmx = np.logical_and( \
np.logical_and((cim == mx), (cim > thresh)), \
bordermask); #%Peter-Kovesi
#% cimmx = (cim==mx) & (cim~=mx2) & (cim>thresh) & bordermask; %my modification
common.DebugPrint("my_nms(): cimmx.shape = %s" % str(cimmx.shape));
#common.DebugPrint("my_nms(): cimmx = %s" % str(cimmx));
#[r,c] = find(cimmx) #% Find row,col coords.
if True:
"""
Retrieving indices in "Fortran" (column-major) order, like in Matlab.
We do this in order to get the harris points sorted like in Matlab.
"""
c, r = np.nonzero(cimmx.T);
else:
# Retrieving indices in row-major order.
r, c = np.nonzero(cimmx);
common.DebugPrint("my_nms(): r = %s" % str(r));
common.DebugPrint("my_nms(): c = %s" % str(c));
if subPixel: #% Compute local maxima to sub pixel accuracy
# From Matlab help: "Determine whether array is empty" "An empty array has at least one dimension of size zero, for example, 0-by-0 or 0-by-5."
#if not isempty(r): #% ...if we have some ponts to work with
if r.size > 0: #% ...if we have some ponts to work with
#ind = sub2ind(size(cim),r,c); #% 1D indices of feature points
ind = Matlab.sub2ind(cim.shape, r, c); #% 1D indices of feature points
w = 1; #% Width that we look out on each side of the feature
#% point to fit a local parabola
if common.MY_DEBUG_STDOUT:
common.DebugPrint("my_nms(): ind.shape = %s" % str(ind.shape));
common.DebugPrint("my_nms(): ind = %s" % str(ind));
common.DebugPrint("my_nms(): ind - w = %s" % str(ind - w));
# Don't forget that we are now in column major ('F'/Fortran) order
#% Indices of points above, below, left and right of feature point
#indrminus1 = max(ind-w,1);
#indrminus1 = np.max(ind - w, 0);
# In Matlab this is what it returns for 1D ind:
assert ind.ndim == 1;
indrminus1 = ind - w;
#indrplus1 = min(ind+w,rows*cols);
#indrplus1 = np.min(ind + w, rows * cols);
# In Matlab this is what it returns for 1D ind:
assert ind.ndim == 1;
indrplus1 = ind + w;
#indcminus1 = max(ind-w*rows,1);
#indcminus1 = np.max(ind - w * rows, 1);
# In Matlab this is what it returns for 1D ind:
assert ind.ndim == 1;
indcminus1 = ind - w * rows;
#indcplus1 = min(ind+w*rows,rows*cols);
#indcplus1 = np.min(ind + w * rows, rows * cols);
# In Matlab this is what it returns for 1D ind:
assert ind.ndim == 1;
indcplus1 = ind + w * rows;
# De-flattening the index arrays back into tuple, as accepted by numpy
# See http://docs.scipy.org/doc/numpy/reference/generated/numpy.unravel_index.html
ind = np.unravel_index(ind, cim.shape, order="F");
indrminus1 = np.unravel_index(indrminus1, cim.shape, order="F");
indrplus1 = np.unravel_index(indrplus1, cim.shape, order="F");
indcminus1 = np.unravel_index(indcminus1, cim.shape, order="F");
indcplus1 = np.unravel_index(indcplus1, cim.shape, order="F");
#% Solve for quadratic down rows
#cy = cim(ind);
cy = cim[ind];
# In Matlab ay has float elements
ay = (cim[indrminus1] + cim[indrplus1]) / 2.0 - cy;
#by = ay + cy - cim(indrminus1);
by = ay + cy - cim[indrminus1];
#rowshift = -w*by./(2*ay); #% Maxima of quadradic
rowshift = -w * by / (2.0 * ay); #% Maxima of quadradic
#% Solve for quadratic across columns
#cx = cim(ind);
cx = cim[ind];
#ax = (cim(indcminus1) + cim(indcplus1))/2 - cx;
ax = (cim[indcminus1] + cim[indcplus1]) / 2.0 - cx;
#bx = ax + cx - cim(indcminus1);
bx = ax + cx - cim[indcminus1];
#colshift = -w*bx./(2*ax); #% Maxima of quadratic
colshift = -w * bx / (2.0 * ax); #% Maxima of quadratic
rsubp = r + rowshift; #% Add subpixel corrections to original row
csubp = c + colshift; #% and column coords.
else:
#rsubp = []; csubp = [];
rsubp = np.array([]);
csubp = np.array([]);
"""
% if nargin==4 & ~isempty(r) % Overlay corners on supplied image.
% figure, imshow(im,[]), hold on
% if subPixel
% plot(csubp,rsubp,'r+'), title('corners detected');
% else
% plot(c,r,'r+'), title('corners detected');
% end
% end
"""
return r, c, rsubp, csubp;
