def get_kpt_and_patch(img_paths):
"""
Detect SIFT keypoints and crop image patches.
Args:
img_paths: a list of image paths.
Returns:
all_patches: Image patches (Nx32x32).
all_npy_kpt: NumPy array, normalized keypoint coordinates ([-1, 1]).
all_cv_kpt: OpenCV KeyPoint, unnormalized keypoint coordinates.
all_sift_desc: SIFT features (Nx128).
all_imgs: RGB images.
"""
sift_wrapper = SiftWrapper(n_sample=FLAGS.max_kpt_num, peak_thld=0.04)
sift_wrapper.standardize = False # the network has handled this step.
sift_wrapper.create()
all_patches = []
all_npy_kpts = []
all_cv_kpts = []
all_sift_desc = []
all_imgs = []
for img_path in img_paths:
img = cv2.imread(img_path)
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
img = img[..., ::-1]
npy_kpts, cv_kpts = sift_wrapper.detect(gray)
sift_desc = sift_wrapper.compute(gray, cv_kpts)
kpt_xy = np.stack(
((npy_kpts[:, 0] - gray.shape[1] / 2) / (gray.shape[1] / 2),
(npy_kpts[:, 1] - gray.shape[0] / 2) / (gray.shape[0] / 2)),
axis=-1)
sift_wrapper.build_pyramid(gray)
patches = sift_wrapper.get_patches(cv_kpts)
all_patches.append(patches)
all_npy_kpts.append(kpt_xy)
all_cv_kpts.append(cv_kpts)
all_sift_desc.append(sift_desc)
all_imgs.append(img)
return all_patches, all_npy_kpts, all_cv_kpts, all_sift_desc, all_imgs
class LocModel(BaseModel):
output_tensors = ["conv6_feat:0", "kpt_mb:0"]
default_config = {
'n_feature': 0,
"n_sample": 0,
'batch_size': 512,
'sift_wrapper': None,
'upright': False,
'scale_diff': False,
'dense_desc': False,
'sift_desc': False,
'peak_thld': 0.0067,
'max_dim': 1280
}
def _init_model(self):
self.sift_wrapper = SiftWrapper(n_feature=self.config['n_feature'],
n_sample=self.config['n_sample'],
peak_thld=self.config['peak_thld'])
self.sift_wrapper.standardize = False # the network has handled this step.
self.sift_wrapper.ori_off = self.config['upright']
self.sift_wrapper.pyr_off = not self.config['scale_diff']
self.sift_wrapper.create()
def _run(self, data):
def _worker(patch_queue, sess, loc_feat, kpt_mb):
"""The worker thread."""
while True:
patch_data = patch_queue.get()
if patch_data is None:
return
loc_returns = sess.run(
self.output_tensors,
feed_dict={"input:0": np.expand_dims(patch_data, -1)})
loc_feat.append(loc_returns[0])
kpt_mb.append(loc_returns[1])
patch_queue.task_done()
assert data.shape[-1] == 1
gray_img = np.squeeze(data, axis=-1).astype(np.uint8)
# detect SIFT keypoints.
npy_kpts, cv_kpts = self.sift_wrapper.detect(gray_img)
if self.config['sift_desc']:
sift_desc = self.sift_wrapper.compute(gray_img, cv_kpts)
else:
sift_desc = None
kpt_xy = np.stack(((npy_kpts[:, 0] - gray_img.shape[1] / 2.) /
(gray_img.shape[1] / 2.),
(npy_kpts[:, 1] - gray_img.shape[0] / 2.) /
(gray_img.shape[0] / 2.)),
axis=-1)
num_patch = len(cv_kpts)
if not self.config['dense_desc']:
self.sift_wrapper.build_pyramid(gray_img)
all_patches = self.sift_wrapper.get_patches(cv_kpts)
# get iteration number
batch_size = self.config['batch_size']
if num_patch % batch_size > 0:
loop_num = int(np.floor(float(num_patch) / float(batch_size)))
else:
loop_num = int(num_patch / batch_size - 1)
# create input thread
loc_feat = []
kpt_mb = []
patch_queue = Queue()
worker_thread = Thread(target=_worker,
args=(patch_queue, self.sess, loc_feat,
kpt_mb))
worker_thread.daemon = True
worker_thread.start()
# enqueue
for i in range(loop_num + 1):
if i < loop_num:
patch_queue.put(all_patches[i * batch_size:(i + 1) *
batch_size])
else:
patch_queue.put(all_patches[i * batch_size:])
# poison pill
patch_queue.put(None)
# wait for extraction.
worker_thread.join()
loc_feat = np.concatenate(loc_feat, axis=0)
kpt_mb = np.concatenate(kpt_mb, axis=0)
else:
import cv2
# compose affine crop matrix.
patch_scale = 6
patch_param = np.zeros((num_patch, 6))
m_cos = np.cos(npy_kpts[:, 3]) * patch_scale * npy_kpts[:, 2]
m_sin = np.sin(npy_kpts[:, 3]) * patch_scale * npy_kpts[:, 2]
short_side = float(min(gray_img.shape[0], gray_img.shape[1]))
patch_param[:, 0] = m_cos / short_side
patch_param[:, 1] = m_sin / short_side
patch_param[:, 2] = kpt_xy[:, 0]
patch_param[:, 3] = -m_sin / short_side
patch_param[:, 4] = m_cos / short_side
patch_param[:, 5] = kpt_xy[:, 1]
max_dim = max(gray_img.shape[0], gray_img.shape[1])
if max_dim > self.config['max_dim']:
downsample_ratio = self.config['max_dim'] / float(max_dim)
gray_img = cv2.resize(gray_img, (0, 0),
fx=downsample_ratio,
fy=downsample_ratio)
gray_img = gray_img[..., np.newaxis]
input_dict = {
"input/img:0": np.expand_dims(gray_img, 0),
"input/kpt_param:0": np.expand_dims(patch_param, 0)
}
local_returns = self.sess.run(self.output_tensors,
feed_dict=input_dict)
loc_feat = local_returns[0]
kpt_mb = local_returns[1]
return loc_feat, kpt_mb, kpt_xy, cv_kpts, sift_desc
def _construct_network(self):
"""Model for patch description."""
if self.config['dense_desc']:
with tf.name_scope('input'):
ph_imgs = tf.placeholder(dtype=tf.float32,
shape=(None, None, None, 1),
name='img')
ph_kpt_params = tf.placeholder(tf.float32,
shape=(None, None, 6),
name='kpt_param')
kpt_xy = tf.concat(
(ph_kpt_params[:, :, 2, None], ph_kpt_params[:, :, 5, None]),
axis=-1)
kpt_theta = tf.reshape(
ph_kpt_params,
(tf.shape(ph_kpt_params)[0], tf.shape(ph_kpt_params)[1], 2, 3))
mean, variance = tf.nn.moments(tf.cast(ph_imgs, tf.float32),
axes=[1, 2],
keep_dims=True)
norm_input = tf.nn.batch_normalization(ph_imgs, mean, variance,
None, None, 1e-5)
config_dict = {}
config_dict['pert_theta'] = kpt_theta
config_dict['patch_sampler'] = transformer_crop
tower = DenseGeoDesc({
'data': norm_input,
'kpt_coord': kpt_xy
},
is_training=False,
resue=False,
**config_dict)
else:
input_size = (32, 32)
patches = tf.placeholder(dtype=tf.float32,
shape=(None, input_size[0], input_size[1],
1),
name='input')
# patch standardization
mean, variance = tf.nn.moments(tf.cast(patches, tf.float32),
axes=[1, 2],
keep_dims=True)
patches = tf.nn.batch_normalization(patches, mean, variance, None,
None, 1e-5)
tower = GeoDesc({'data': patches}, is_training=False, reuse=False)
conv6_feat = tower.get_output_by_name('conv6')
conv6_feat = tf.squeeze(conv6_feat, axis=[1, 2], name='conv6_feat')
with tf.variable_scope('kpt_m'):
inter_feat = tower.get_output_by_name('conv5')
matchability_tower = MatchabilityPrediction({'data': inter_feat},
is_training=False,
reuse=False)
mb = matchability_tower.get_output()
mb = tf.squeeze(mb, axis=[1, 2], name='kpt_mb')
def get_data(self, seq_idx, ori_est, dense_desc):
random.seed(0)
if self.suffix is None:
sift_wrapper = SiftWrapper(n_feature=self.sample_num,
peak_thld=0.04)
sift_wrapper.create()
hseq_data = HSeqData()
seq_name = self.seqs[seq_idx]
for img_idx in range(1, 7):
# read image features.
img_feat = np.load(
os.path.join(seq_name, '%d_img_feat.npy' % img_idx))
rows = img_feat.shape[0]
cols = img_feat.shape[1]
x_rng = np.linspace(-1., 1., cols)
y_rng = np.linspace(-1., 1., rows)
xv, yv = np.meshgrid(x_rng, y_rng)
grid_pts = np.stack((xv, yv), axis=-1)
# read images.
img = cv2.imread(os.path.join(seq_name, '%d.ppm' % img_idx))
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
img_size = img.shape
if self.suffix is None:
npy_kpts, cv_kpts = sift_wrapper.detect(gray)
if not dense_desc:
sift_wrapper.build_pyramid(gray)
patches = sift_wrapper.get_patches(cv_kpts)
else:
patches = None
else:
with open(
os.path.join(seq_name,
('%d' + self.suffix + '.pkl') % img_idx),
'rb') as handle:
data_dict = pickle.load(handle, encoding='latin1')
npy_kpts = data_dict['npy_kpts']
if not dense_desc:
patches = data_dict['patches']
else:
patches = None
kpt_num = npy_kpts.shape[0]
# Sample keypoints
if self.sample_num > 0 and kpt_num > self.sample_num:
sample_idx = random.sample(range(kpt_num), self.sample_num)
else:
sample_idx = range(kpt_num)
# Apply sampling.
npy_kpts = npy_kpts[sample_idx]
if patches is not None:
patches = patches[sample_idx]
kpt_num = npy_kpts.shape[0]
# compose affine crop matrix.
crop_mat = np.zeros((kpt_num, 6))
if ori_est:
# no initial orientation.
m_cos = np.ones_like(
npy_kpts[:, 2]) * self.patch_scale * npy_kpts[:, 2]
m_sin = np.zeros_like(
npy_kpts[:, 2]) * self.patch_scale * npy_kpts[:, 2]
else:
# rely on the SIFT orientation estimation.
m_cos = np.cos(
-npy_kpts[:, 3]) * self.patch_scale * npy_kpts[:, 2]
m_sin = np.sin(
-npy_kpts[:, 3]) * self.patch_scale * npy_kpts[:, 2]
crop_mat[:, 0] = m_cos / float(img_size[1])
crop_mat[:, 1] = m_sin / float(img_size[1])
crop_mat[:, 2] = (npy_kpts[:, 0] -
img_size[1] / 2.) / (img_size[1] / 2.)
crop_mat[:, 3] = -m_sin / float(img_size[0])
crop_mat[:, 4] = m_cos / float(img_size[0])
crop_mat[:, 5] = (npy_kpts[:, 1] -
img_size[0] / 2.) / (img_size[0] / 2.)
npy_kpts = npy_kpts[:, 0:2]
# read homography matrix.
if img_idx > 1:
homo_mat = open(os.path.join(seq_name, 'H_1_%d' %
img_idx)).read().splitlines()
homo_mat = np.array(
[float(i) for i in ' '.join(homo_mat).split()])
homo_mat = np.reshape(homo_mat, (3, 3))
else:
homo_mat = None
hseq_data.img.append(img)
hseq_data.kpt_param.append(crop_mat)
hseq_data.patch.append(patches)
hseq_data.coord.append(npy_kpts)
hseq_data.h**o.append(homo_mat)
hseq_data.img_feat.append((img_feat, grid_pts))
return seq_name, hseq_data
class GeodescModel(BaseModel):
output_tensors = ["squeeze_1:0"]
default_config = {'n_feature': 0, "n_sample": 0,
'batch_size': 512, 'sift_wrapper': None, 'upright': False, 'scale_diff': False,
'dense_desc': False, 'sift_desc': False, 'peak_thld': 0.0067, 'edge_thld': 10, 'max_dim': 1280}
def _init_model(self):
self.sift_wrapper = SiftWrapper(
n_feature=self.config['n_feature'],
n_sample=self.config['n_sample'],
peak_thld=self.config['peak_thld'],
edge_thld=self.config['edge_thld']
)
self.sift_wrapper.standardize = True
self.sift_wrapper.ori_off = self.config['upright']
self.sift_wrapper.pyr_off = not self.config['scale_diff']
self.sift_wrapper.create()
def _run(self, data, **kwargs):
def _worker(patch_queue, sess, loc_feat):
"""The worker thread."""
while True:
patch_data = patch_queue.get()
if patch_data is None:
return
loc_returns = sess.run(self.output_tensors,
feed_dict={"input:0": np.expand_dims(patch_data, -1)})
loc_returns = loc_returns[0]
if len(loc_returns.shape) == 1:
loc_returns = np.expand_dims(loc_returns, axis=0)
loc_feat.append(loc_returns)
patch_queue.task_done()
gray_img = np.squeeze(data, axis=-1).astype(np.uint8)
# detect SIFT keypoints.
npy_kpts, cv_kpts = self.sift_wrapper.detect(gray_img)
num_patch = len(cv_kpts)
self.sift_wrapper.build_pyramid(gray_img)
all_patches = self.sift_wrapper.get_patches(cv_kpts)
# get iteration number
batch_size = self.config['batch_size']
if num_patch % batch_size > 0:
loop_num = int(np.floor(float(num_patch) / float(batch_size)))
else:
loop_num = int(num_patch / batch_size - 1)
# create input thread
loc_feat = []
patch_queue = Queue()
worker_thread = Thread(target=_worker, args=(patch_queue, self.sess, loc_feat))
worker_thread.daemon = True
worker_thread.start()
# enqueue
for i in range(loop_num + 1):
if i < loop_num:
patch_queue.put(all_patches[i * batch_size: (i + 1) * batch_size])
else:
patch_queue.put(all_patches[i * batch_size:])
# poison pill
patch_queue.put(None)
# wait for extraction.
worker_thread.join()
loc_feat = np.concatenate(loc_feat, axis=0)
loc_feat = (loc_feat * 128 + 128)
return npy_kpts, loc_feat
def _construct_network(self):
"""Model for patch description."""
return