# Modify the network so that we bypass the keypoint part and the
# descriptor part.
param.model.sDetector = 'bypass'
# This ensures that you don't create unecessary scale space
param.model.fScaleList = np.array([1.0])
param.patch.fMaxScale = np.max(param.model.fScaleList)
# this ensures that you don't over eliminate features at boundaries
param.model.nPatchSize = int(np.round(param.model.nDescInputSize) *
np.sqrt(2))
param.patch.fRatioScale = (float(param.patch.nPatchSize) /
float(param.model.nDescInputSize)) * 6.0
param.model.sDescriptor = 'bypass'
# add the mean and std of the learned model to the param
mean_std_file = pathconf.result + 'mean_std.h5'
mean_std_dict = loadh5(mean_std_file)
param.online = paramGroup()
setattr(param.online, 'mean_x', mean_std_dict['mean_x'])
setattr(param.online, 'std_x', mean_std_dict['std_x'])
# -------------------------------------------------------------------------
# Load data in the test format
test_data_in = data_module.data_obj(param, image_file_name, kp_file_name)
# -------------------------------------------------------------------------
# Test using the test function
start_time = time.clock()
_, oris, compile_time = Test(
pathconf, param, test_data_in, test_mode="ori")
end_time = time.clock()
compute_time = (end_time - start_time) * 1000.0 - compile_time
# Modify the network so that we bypass the keypoint part and the
# orientation part
param.model.sDetector = 'bypass'
# This ensures that you don't create unecessary scale space
param.model.fScaleList = np.array([1.0])
param.patch.fMaxScale = np.max(param.model.fScaleList)
# this ensures that you don't over eliminate features at boundaries
param.model.nPatchSize = int(
np.round(param.model.nDescInputSize) * np.sqrt(2))
param.patch.fRatioScale = (float(param.patch.nPatchSize) /
float(param.model.nDescInputSize)) * 6.0
param.model.sOrientation = 'bypass'
# add the mean and std of the learned model to the param
mean_std_file = pathconf.result + 'mean_std.h5'
mean_std_dict = loadh5(mean_std_file)
param.online = paramGroup()
setattr(param.online, 'mean_x', mean_std_dict['mean_x'])
setattr(param.online, 'std_x', mean_std_dict['std_x'])
# -------------------------------------------------------------------------
# Load data in the test format
test_data_in = data_module.data_obj(param, image_file_name, kp_file_name)
# -------------------------------------------------------------------------
# Test using the test function
start_time = time.clock()
descs, _, compile_time = Test(pathconf,
param,
test_data_in,
test_mode="desc")
def build(myNet, idxSiam, verbose=True):
# Load model
fn = '%s/%s' % (myNet.config.descriptor_export_folder,
myNet.config.descriptor_model)
model_dict = dt.loadh5(fn)
# Load training mean/std
# if we have the normalization setup
if myNet.config.bNormalizeInput:
kwang_mean = np.cast[floatX](myNet.config.mean_x)
kwang_std = np.cast[floatX](myNet.config.std_x)
# else, simply divide with 255
else:
kwang_mean = np.cast[floatX](0.0)
kwang_std = np.cast[floatX](255.0)
if 'patch-mean' in model_dict.keys():
desc_mean_x = np.cast[floatX](model_dict['patch-mean'][0])
desc_std_x = np.cast[floatX](model_dict['patch-std'][0])
else:
print('Warning: no mean/std in the model file')
desc_mean_x = kwang_mean
desc_std_x = kwang_std
# Layer indices
indices = model_dict['layers']
if verbose and idxSiam == 0:
print('*** Loading descriptor "%s" ***' % fn)
print('Number of elements: %d' % indices.size)
# Add another layer that transforms the original input
curr_name = 'desc-re-normalize'
curr_input = myNet.config.descriptor_input
myNet.layers[idxSiam][curr_name] = ExpressionLayer(
myNet.layers[idxSiam][curr_input],
lambda x: (x * kwang_std + kwang_mean - desc_mean_x) / desc_std_x,
name=curr_name)
curr_input = curr_name
# Loop over layers
for i in six.moves.xrange(indices.size):
if indices[i] == 1:
if verbose and idxSiam == 0:
print('%d -> SpatialConvolution' % i)
curr_name = 'desc-%d-conv' % (i + 1)
# read actual value for siamese 0
w = model_dict['l-%d-weights' % (i + 1)].astype(floatX)
b = model_dict['l-%d-bias' % (i + 1)].astype(floatX)
num_filters, num_input_channels, filter_size, filter_size = w.shape
# assert num_input_channels == myNet.config.nDescInputChannels
# assert filter_size == myNet.config.nDescInputSize
if verbose and idxSiam == 0:
print(' Number of filters: %d' % num_filters)
print(' Filter size: %d' % filter_size)
# Manually create shared variables
if idxSiam == 0:
w = theano.shared(w, name=curr_name + '.W')
b = theano.shared(b, name=curr_name + '.b')
else:
w = myNet.layers[0][curr_name].W
b = myNet.layers[0][curr_name].b
myNet.layers[idxSiam][curr_name] = Conv2DLayer(
myNet.layers[idxSiam][curr_input],
num_filters,
filter_size,
W=w,
b=b,
nonlinearity=None, # no activation
flip_filters=False,
name=curr_name)
elif indices[i] == 2:
if verbose and idxSiam == 0:
print('%d -> Linear' % i)
raise RuntimeError('Layer type %d TODO' % i)
elif indices[i] == 3:
if verbose and idxSiam == 0:
print('%d -> SpatialMaxPooling' % i)
curr_name = 'desc-%d-maxpool' % (i + 1)
kw = model_dict['l-%d-kw' % (i + 1)].astype(np.int32)[0]
kh = model_dict['l-%d-kh' % (i + 1)].astype(np.int32)[0]
if verbose and idxSiam == 0:
print(' Region size: %dx%d' % (kw, kh))
assert kw == kh
kw = int(kw)
myNet.layers[idxSiam][curr_name] = MaxPool2DLayer(
myNet.layers[idxSiam][curr_input],
pool_size=kw,
stride=None,
name=curr_name)
elif indices[i] == 4:
if verbose and idxSiam == 0:
print('%d -> SpatialLPPooling' % i)
curr_name = 'desc-%d-lppool' % (i + 1)
kw = model_dict['l-%d-kw' % (i + 1)].astype(np.int32)[0]
kh = model_dict['l-%d-kh' % (i + 1)].astype(np.int32)[0]
if verbose and idxSiam == 0:
print(' Region size: %dx%d' % (kw, kh))
assert kw == kh
kw = int(kw)
myNet.layers[idxSiam][curr_name] = LPPool2DLayer(
myNet.layers[idxSiam][curr_input],
pnorm=2,
pool_size=kw,
stride=None,
name=curr_name)
elif indices[i] == 5:
if verbose and idxSiam == 0:
print('%d -> Tanh' % i)
curr_name = 'desc-%d-tanh' % (i + 1)
myNet.layers[idxSiam][curr_name] = NonlinearityLayer(
myNet.layers[idxSiam][curr_input],
nonlinearity=nonlinearities.tanh,
name=curr_name)
elif indices[i] == 6:
if verbose and idxSiam == 0:
print('%d -> ReLU' % i)
curr_name = 'desc-%d-relu' % (i + 1)
myNet.layers[idxSiam][curr_name] = NonlinearityLayer(
myNet.layers[idxSiam][curr_input],
nonlinearity=nonlinearities.rectify,
name=curr_name)
elif indices[i] == 7:
if verbose and idxSiam == 0:
print('%d -> SpatialSubtractiveNormalization' % i)
curr_name = 'desc-%d-subt-norm' % (i + 1)
kernel = model_dict['l-%d-filter' % (i + 1)].astype(floatX)
w_kernel, h_kernel = kernel.shape
if verbose and idxSiam == 0:
print(' Kernel size: %dx%d' % (w_kernel, h_kernel))
myNet.layers[idxSiam][curr_name] = SubtractiveNormalization2DLayer(
myNet.layers[idxSiam][curr_input],
kernel=kernel,
name=curr_name)
else:
raise RuntimeError('Layer type %d: unknown' % i)
# Input to the next layer
curr_input = curr_name
# Flatten output and rename
myNet.layers[idxSiam]['desc-output'] = FlattenLayer(
myNet.layers[idxSiam][curr_input], outdim=2)
def apply_learned_filter_2_image_no_theano(image, save_dir,
bNormalizeInput,
sKpNonlinearity,
verbose=True,
network_weights=None):
"""Apply the learned keypoint detector to image.
Parameters
----------
image: np.ndarray
Raw image, without any pre-processing.
save_dir: str
Full path to the model directory.
bNormalizeInput: bool
Normalization parameter used for training.
sKpNonlinearity: str
Nonlinearity applied to the keypoint scoremap.
"""
if len(image.shape) > 2:
raise NotImplementedError("This function is only meant for "
"grayscale images!")
num_in_sum = 4
num_in_max = 4
if network_weights is None:
# Load model
if os.path.exists(save_dir + 'model.h5'):
model = loadh5(save_dir + 'model.h5')
W = model['kp-c0']['kp-c0.W'].astype(floatX)
b = model['kp-c0']['kp-c0.b'].astype(floatX)
elif os.path.exists(save_dir + 'model.pklz'):
model = loadpklz(save_dir + 'model.pklz')
W = model[0].astype(floatX)
b = model[1].astype(floatX)
else:
raise RuntimeError('Learned model does not exist!')
else:
model = loadh5(network_weights)
W = model['kp-c0']['kp-c0.W'].astype(floatX)
b = model['kp-c0']['kp-c0.b'].astype(floatX)
# add a new axis at the end if the filter is 4D
if W.ndim == 4:
W = W[..., np.newaxis]
if bNormalizeInput:
print("Using trained mean and Std for normalization")
mean_std_dict = loadh5(save_dir + 'mean_std.h5')
image_prep = ((image - mean_std_dict['mean_x']) /
mean_std_dict['std_x'])
else:
print("Just dividing with 255!")
image_prep = image / np.cast[floatX](255.0)
# Do subtracive normalization if it is there
bSubtractiveNormalization = False
if 'kp-subnorm' in model.keys():
norm_kern = model['kp-subnorm']['kp-subnorm.kernel']
assert np.isclose(np.sum(norm_kern), 1.0)
bSubtractiveNormalization = True
# prepare image
if bSubtractiveNormalization:
print("Performing subtractive normalization!")
# compute the coeffs to make adjust at borders
coef = _subtractive_norm_make_coef(norm_kern, image.shape[:2])
# run the filter to get means
conv_mean = cv2.filter2D(image_prep, -1, norm_kern,
borderType=cv2.BORDER_CONSTANT)
# adjust means with precomputed coef
adj_mean = conv_mean / coef
# subtract the mean
image_prep -= adj_mean
# Do LCN if it is there
bLCN = False
if 'kp-lcn' in model.keys():
norm_kern = model['kp-lcn']['kp-lcn.kernel']
assert np.isclose(np.sum(norm_kern), 1.0)
bLCN = True
if bLCN:
# compute the coeffs to make adjust at borders
coef = _lcn_make_coef(norm_kern, image.shape[:2])
# run the filter to get means and adjust
conv_mean = cv2.filter2D(image_prep, -1, norm_kern,
borderType=cv2.BORDER_CONSTANT)
adj_mean = conv_mean / coef
# subtract the mean
sub_norm = image_prep - adj_mean
# run the filter to get std and adjust
conv_std = np.sqrt(
cv2.filter2D(sub_norm**2.0, -1, norm_kern,
borderType=cv2.BORDER_CONSTANT)
)
adj_std = np.sqrt(conv_std / coef)
# run the filter once more to get std mean and adjust it
conv_mean_std = np.sqrt(
cv2.filter2D(adj_std, -1, norm_kern,
borderType=cv2.BORDER_CONSTANT)
)
adj_mean_std = conv_mean_std / coef
# now divide
image_prep = sub_norm / np.maximum(adj_mean_std, adj_std)
h, w = image_prep.shape
# NEW implementation
nonlinearity = sKpNonlinearity
resp = apply_ghh_filter(
image_prep, W, b,
num_in_sum, num_in_max, nonlinearity)
# The final return of this function should be a single channel image
if len(resp.shape) == 3:
assert resp.shape[2] == 1
resp = np.squeeze(resp)
return resp
def apply_learned_filter_2_image_no_theano(image,
save_dir,
bNormalizeInput,
sKpNonlinearity,
verbose=True,
network_weights=None):
"""Apply the learned keypoint detector to image.
Parameters
----------
image: np.ndarray
Raw image, without any pre-processing.
save_dir: str
Full path to the model directory.
bNormalizeInput: bool
Normalization parameter used for training.
sKpNonlinearity: str
Nonlinearity applied to the keypoint scoremap.
"""
if len(image.shape) > 2:
raise NotImplementedError("This function is only meant for "
"grayscale images!")
num_in_sum = 4
num_in_max = 4
if network_weights is None:
# Load model
if os.path.exists(save_dir + 'model.h5'):
model = loadh5(save_dir + 'model.h5')
W = model['kp-c0']['kp-c0.W'].astype(floatX)
b = model['kp-c0']['kp-c0.b'].astype(floatX)
elif os.path.exists(save_dir + 'model.pklz'):
model = loadpklz(save_dir + 'model.pklz')
W = model[0].astype(floatX)
b = model[1].astype(floatX)
else:
raise RuntimeError('Learned model does not exist!')
else:
model = loadh5(network_weights)
W = model['kp-c0']['kp-c0.W'].astype(floatX)
b = model['kp-c0']['kp-c0.b'].astype(floatX)
# add a new axis at the end if the filter is 4D
if W.ndim == 4:
W = W[..., np.newaxis]
if bNormalizeInput:
print("Using trained mean and Std for normalization")
mean_std_dict = loadh5(save_dir + 'mean_std.h5')
image_prep = ((image - mean_std_dict['mean_x']) /
mean_std_dict['std_x'])
else:
print("Just dividing with 255!")
image_prep = image / np.cast[floatX](255.0)
# Do subtracive normalization if it is there
bSubtractiveNormalization = False
if 'kp-subnorm' in model.keys():
norm_kern = model['kp-subnorm']['kp-subnorm.kernel']
assert np.isclose(np.sum(norm_kern), 1.0)
bSubtractiveNormalization = True
# prepare image
if bSubtractiveNormalization:
print("Performing subtractive normalization!")
# compute the coeffs to make adjust at borders
coef = _subtractive_norm_make_coef(norm_kern, image.shape[:2])
# run the filter to get means
conv_mean = cv2.filter2D(image_prep,
-1,
norm_kern,
borderType=cv2.BORDER_CONSTANT)
# adjust means with precomputed coef
adj_mean = conv_mean / coef
# subtract the mean
image_prep -= adj_mean
# Do LCN if it is there
bLCN = False
if 'kp-lcn' in model.keys():
norm_kern = model['kp-lcn']['kp-lcn.kernel']
assert np.isclose(np.sum(norm_kern), 1.0)
bLCN = True
if bLCN:
# compute the coeffs to make adjust at borders
coef = _lcn_make_coef(norm_kern, image.shape[:2])
# run the filter to get means and adjust
conv_mean = cv2.filter2D(image_prep,
-1,
norm_kern,
borderType=cv2.BORDER_CONSTANT)
adj_mean = conv_mean / coef
# subtract the mean
sub_norm = image_prep - adj_mean
# run the filter to get std and adjust
conv_std = np.sqrt(
cv2.filter2D(sub_norm**2.0,
-1,
norm_kern,
borderType=cv2.BORDER_CONSTANT))
adj_std = np.sqrt(conv_std / coef)
# run the filter once more to get std mean and adjust it
conv_mean_std = np.sqrt(
cv2.filter2D(adj_std,
-1,
norm_kern,
borderType=cv2.BORDER_CONSTANT))
adj_mean_std = conv_mean_std / coef
# now divide
image_prep = sub_norm / np.maximum(adj_mean_std, adj_std)
h, w = image_prep.shape
# NEW implementation
nonlinearity = sKpNonlinearity
resp = apply_ghh_filter(image_prep, W, b, num_in_sum, num_in_max,
nonlinearity)
# The final return of this function should be a single channel image
if len(resp.shape) == 3:
assert resp.shape[2] == 1
resp = np.squeeze(resp)
return resp
def build(myNet, idxSiam, verbose=True):
# Load model
fn = '%s/%s' % (myNet.config.descriptor_export_folder,
myNet.config.descriptor_model)
model_dict = dt.loadh5(fn)
# Load training mean/std
# if we have the normalization setup
if myNet.config.bNormalizeInput:
kwang_mean = np.cast[floatX](myNet.config.mean_x)
kwang_std = np.cast[floatX](myNet.config.std_x)
# else, simply divide with 255
else:
kwang_mean = np.cast[floatX](0.0)
kwang_std = np.cast[floatX](255.0)
if 'patch-mean' in model_dict.keys():
desc_mean_x = np.cast[floatX](model_dict['patch-mean'][0])
desc_std_x = np.cast[floatX](model_dict['patch-std'][0])
else:
print('Warning: no mean/std in the model file')
desc_mean_x = kwang_mean
desc_std_x = kwang_std
# Layer indices
indices = model_dict['layers']
if verbose and idxSiam == 0:
print('*** Loading descriptor "%s" ***' % fn)
print('Number of elements: %d' % indices.size)
# Add another layer that transforms the original input
curr_name = 'desc-re-normalize'
curr_input = myNet.config.descriptor_input
myNet.layers[idxSiam][curr_name] = ExpressionLayer(
myNet.layers[idxSiam][curr_input],
lambda x: (x * kwang_std + kwang_mean - desc_mean_x) / desc_std_x,
name=curr_name
)
curr_input = curr_name
# Loop over layers
for i in six.moves.xrange(indices.size):
if indices[i] == 1:
if verbose and idxSiam == 0:
print('%d -> SpatialConvolution' % i)
curr_name = 'desc-%d-conv' % (i + 1)
# read actual value for siamese 0
w = model_dict['l-%d-weights' % (i + 1)].astype(floatX)
b = model_dict['l-%d-bias' % (i + 1)].astype(floatX)
num_filters, num_input_channels, filter_size, filter_size = w.shape
# assert num_input_channels == myNet.config.nDescInputChannels
# assert filter_size == myNet.config.nDescInputSize
if verbose and idxSiam == 0:
print(' Number of filters: %d' % num_filters)
print(' Filter size: %d' % filter_size)
# Manually create shared variables
if idxSiam == 0:
w = theano.shared(w, name=curr_name + '.W')
b = theano.shared(b, name=curr_name + '.b')
else:
w = myNet.layers[0][curr_name].W
b = myNet.layers[0][curr_name].b
myNet.layers[idxSiam][curr_name] = Conv2DLayer(
myNet.layers[idxSiam][curr_input],
num_filters,
filter_size,
W=w,
b=b,
nonlinearity=None, # no activation
flip_filters=False,
name=curr_name
)
elif indices[i] == 2:
if verbose and idxSiam == 0:
print('%d -> Linear' % i)
raise RuntimeError('Layer type %d TODO' % i)
elif indices[i] == 3:
if verbose and idxSiam == 0:
print('%d -> SpatialMaxPooling' % i)
curr_name = 'desc-%d-maxpool' % (i + 1)
kw = model_dict['l-%d-kw' % (i + 1)].astype(np.int32)[0]
kh = model_dict['l-%d-kh' % (i + 1)].astype(np.int32)[0]
if verbose and idxSiam == 0:
print(' Region size: %dx%d' % (kw, kh))
assert kw == kh
kw = int(kw)
myNet.layers[idxSiam][curr_name] = MaxPool2DLayer(
myNet.layers[idxSiam][curr_input],
pool_size=kw,
stride=None,
name=curr_name
)
elif indices[i] == 4:
if verbose and idxSiam == 0:
print('%d -> SpatialLPPooling' % i)
curr_name = 'desc-%d-lppool' % (i + 1)
kw = model_dict['l-%d-kw' % (i + 1)].astype(np.int32)[0]
kh = model_dict['l-%d-kh' % (i + 1)].astype(np.int32)[0]
if verbose and idxSiam == 0:
print(' Region size: %dx%d' % (kw, kh))
assert kw == kh
kw = int(kw)
myNet.layers[idxSiam][curr_name] = LPPool2DLayer(
myNet.layers[idxSiam][curr_input],
pnorm=2,
pool_size=kw,
stride=None,
name=curr_name
)
elif indices[i] == 5:
if verbose and idxSiam == 0:
print('%d -> Tanh' % i)
curr_name = 'desc-%d-tanh' % (i + 1)
myNet.layers[idxSiam][curr_name] = NonlinearityLayer(
myNet.layers[idxSiam][curr_input],
nonlinearity=nonlinearities.tanh,
name=curr_name
)
elif indices[i] == 6:
if verbose and idxSiam == 0:
print('%d -> ReLU' % i)
curr_name = 'desc-%d-relu' % (i + 1)
myNet.layers[idxSiam][curr_name] = NonlinearityLayer(
myNet.layers[idxSiam][curr_input],
nonlinearity=nonlinearities.rectify,
name=curr_name
)
elif indices[i] == 7:
if verbose and idxSiam == 0:
print('%d -> SpatialSubtractiveNormalization' % i)
curr_name = 'desc-%d-subt-norm' % (i + 1)
kernel = model_dict['l-%d-filter' % (i + 1)].astype(floatX)
w_kernel, h_kernel = kernel.shape
if verbose and idxSiam == 0:
print(' Kernel size: %dx%d' % (w_kernel, h_kernel))
myNet.layers[idxSiam][curr_name] = SubtractiveNormalization2DLayer(
myNet.layers[idxSiam][curr_input],
kernel=kernel,
name=curr_name
)
else:
raise RuntimeError('Layer type %d: unknown' % i)
# Input to the next layer
curr_input = curr_name
# Flatten output and rename
myNet.layers[idxSiam]['desc-output'] = FlattenLayer(
myNet.layers[idxSiam][curr_input],
outdim=2
)
def load_data_for_set(self, pathconf, param,
image_file_name, kp_file_name):
bUseColorImage = getattr(param.patch, "bUseColorImage", False)
if not bUseColorImage:
# If there is not gray image, load the color one and convert to
# gray
if os.path.exists(image_file_name.replace(
"image_color", "image_gray"
)):
img = cv2.imread(image_file_name.replace(
"image_color", "image_gray"
), 0)
assert len(img.shape) == 2
else:
# read the image
img = cv2.cvtColor(cv2.imread(image_file_name),
cv2.COLOR_BGR2GRAY)
in_dim = 1
else:
img = cv2.imread(image_file_name)
in_dim = 3
assert(img.shape[-1] == in_dim)
bTestWithTestMeanStd = getattr(
param.validation, 'bTestWithTestMeanStd', False)
if bTestWithTestMeanStd:
img = img - np.mean(img)
mean_std_dict = loadh5(pathconf.result + 'mean_std.h5')
img = img + mean_std_dict['mean_x']
print("Test image has range {} to {} after "
"transforming to the test domain"
"".format(np.min(img), np.max(img)))
# load the keypoint informations
kp = np.asarray(loadKpListFromTxt(kp_file_name))
# Assign dummy values to y, ID, angle
y = np.zeros((len(kp),))
ID = np.zeros((len(kp),), dtype='int64')
# angle = np.zeros((len(kp),))
angle = np.pi / 180.0 * kp[:, IDX_ANGLE] # store angle in radians
# load patches with id (drop out of boundary)
bPerturb = False
fPerturbInfo = np.zeros((3,))
dataset = load_patches(img, kp, y, ID, angle, param.patch.fRatioScale,
param.patch.fMaxScale, param.patch.nPatchSize,
param.model.nDescInputSize, in_dim, bPerturb,
fPerturbInfo, bReturnCoords=True)
x = dataset[0]
y = dataset[1]
ID = dataset[2]
pos = dataset[3]
angle = dataset[4]
coords = dataset[5]
return x, y, ID, pos, angle, coords