def DelfFeaturePostProcessing(boxes, descriptors, config):
"""Extract DELF features from input image.
Args:
boxes: [N, 4] float tensor which denotes the selected receptive box. N is
the number of final feature points which pass through keypoint selection
and NMS steps.
descriptors: [N, input_dim] float tensor.
config: DelfConfig proto with DELF extraction options.
Returns:
locations: [N, 2] float tensor which denotes the selected keypoint
locations.
final_descriptors: [N, output_dim] float tensor with DELF descriptors after
normalization and (possibly) PCA/whitening.
"""
# Get center of descriptor boxes, corresponding to feature locations.
locations = CalculateKeypointCenters(boxes)
# Post-process descriptors: L2-normalize, and if desired apply PCA (followed
# by L2-normalization).
with tf.variable_scope('postprocess'):
final_descriptors = tf.nn.l2_normalize(descriptors,
dim=1,
name='l2_normalization')
if config.delf_local_config.use_pca:
# Load PCA parameters.
pca_mean = tf.constant(datum_io.ReadFromFile(
config.delf_local_config.pca_parameters.mean_path),
dtype=tf.float32)
pca_matrix = tf.constant(
datum_io.ReadFromFile(config.delf_local_config.pca_parameters.
projection_matrix_path),
dtype=tf.float32)
pca_dim = config.delf_local_config.pca_parameters.pca_dim
pca_variances = None
if config.delf_local_config.pca_parameters.use_whitening:
pca_variances = tf.constant(
datum_io.ReadFromFile(config.delf_local_config.
pca_parameters.pca_variances_path),
dtype=tf.float32)
# Apply PCA, and whitening if desired.
final_descriptors = ApplyPcaAndWhitening(
final_descriptors, pca_matrix, pca_mean, pca_dim,
config.delf_local_config.pca_parameters.use_whitening,
pca_variances)
# Re-normalize.
final_descriptors = tf.nn.l2_normalize(final_descriptors,
dim=1,
name='pca_l2_normalization')
return locations, final_descriptors
def _ReadDelgGlobalDescriptors(input_dir, image_list):
"""Reads DELG global features.
Args:
input_dir: Directory where features are located.
image_list: List of image names for which to load features.
Returns:
global_descriptors: NumPy array of shape (len(image_list), D), where D
corresponds to the global descriptor dimensionality.
"""
num_images = len(image_list)
global_descriptors = []
print('Starting to collect global descriptors for %d images...' %
num_images)
start = time.time()
for i in range(num_images):
if i > 0 and i % _STATUS_CHECK_LOAD_ITERATIONS == 0:
elapsed = (time.time() - start)
print('Reading global descriptors for image %d out of %d, last %d '
'images took %f seconds' %
(i, num_images, _STATUS_CHECK_LOAD_ITERATIONS, elapsed))
start = time.time()
descriptor_filename = image_list[i] + _DELG_GLOBAL_EXTENSION
descriptor_fullpath = os.path.join(input_dir, descriptor_filename)
global_descriptors.append(datum_io.ReadFromFile(descriptor_fullpath))
return np.array(global_descriptors)
def testWriteAndReadToFile(self):
data = np.array([[[-1.0, 125.0, -2.5], [14.5, 3.5, 0.0]],
[[20.0, 0.0, 30.0], [25.5, 36.0, 42.0]]])
filename = os.path.join(FLAGS.test_tmpdir, 'test.datum')
datum_io.WriteToFile(data, filename)
data_read = datum_io.ReadFromFile(filename)
self.assertAllEqual(data_read, data)
def DelfFeaturePostProcessing(boxes, descriptors, config):
""" compute centers of receptive fields; apply PCA + whitening to features """
locations = CalculateKeypointCenters(boxes)
with tf.variable_scope('postprocess'):
final_descriptors = tf.nn.l2_normalize(descriptors,
axis=1,
name='l2_normalization')
if config.delf_local_config.use_pca:
pca_dim = config.delf_local_config.pca_parameters.pca_dim
mean_path = os.path.expanduser(
config.delf_local_config.pca_parameters.mean_path)
pca_mean = tf.constant(datum_io.ReadFromFile(mean_path),
dtype=tf.float32)
projection_matrix_path = os.path.expanduser(
config.delf_local_config.pca_parameters.projection_matrix_path)
pca_matrix = tf.constant(
datum_io.ReadFromFile(projection_matrix_path),
dtype=tf.float32)
if config.delf_local_config.pca_parameters.use_whitening:
pca_variances_path = os.path.expanduser(
config.delf_local_config.pca_parameters.pca_variances_path)
pca_variances = tf.constant(
datum_io.ReadFromFile(pca_variances_path),
dtype=tf.float32)
else:
pca_variances = None
final_descriptors = ApplyPcaAndWhitening(
final_descriptors, pca_matrix, pca_mean, pca_dim,
config.delf_local_config.pca_parameters.use_whitening,
pca_variances)
final_descriptors = tf.nn.l2_normalize(final_descriptors,
axis=1,
name='pca_l2_normalization')
return locations, final_descriptors
def PostProcessDescriptors(descriptors, use_pca, pca_parameters):
"""Post-process descriptors.
Args:
descriptors: [N, input_dim] float tensor.
use_pca: Whether to use PCA.
pca_parameters: DelfPcaParameters proto.
Returns:
final_descriptors: [N, output_dim] float tensor with descriptors after
normalization and (possibly) PCA/whitening.
"""
# L2-normalize, and if desired apply PCA (followed by L2-normalization).
with tf.compat.v1.variable_scope('postprocess'):
final_descriptors = tf.nn.l2_normalize(
descriptors, axis=1, name='l2_normalization')
if use_pca:
# Load PCA parameters.
pca_mean = tf.constant(
datum_io.ReadFromFile(pca_parameters.mean_path), dtype=tf.float32)
pca_matrix = tf.constant(
datum_io.ReadFromFile(pca_parameters.projection_matrix_path),
dtype=tf.float32)
pca_dim = pca_parameters.pca_dim
pca_variances = None
if pca_parameters.use_whitening:
pca_variances = tf.squeeze(
tf.constant(
datum_io.ReadFromFile(pca_parameters.pca_variances_path),
dtype=tf.float32))
# Apply PCA, and whitening if desired.
final_descriptors = ApplyPcaAndWhitening(final_descriptors, pca_matrix,
pca_mean, pca_dim,
pca_parameters.use_whitening,
pca_variances)
# Re-normalize.
final_descriptors = tf.nn.l2_normalize(
final_descriptors, axis=1, name='pca_l2_normalization')
return final_descriptors
def _ReadAggregatedDescriptors(input_dir, image_list, config):
"""Reads aggregated descriptors.
Args:
input_dir: Directory where aggregated descriptors are located.
image_list: List of image names for which to load descriptors.
config: AggregationConfig used for images.
Returns:
aggregated_descriptors: List containing #images items, each a 1D NumPy
array.
visual_words: If using VLAD aggregation, returns an empty list. Otherwise,
returns a list containing #images items, each a 1D NumPy array.
"""
# Compose extension of aggregated descriptors.
extension = '.'
if config.use_regional_aggregation:
extension += 'r'
if config.aggregation_type == _VLAD:
extension += _VLAD_EXTENSION_SUFFIX
elif config.aggregation_type == _ASMK:
extension += _ASMK_EXTENSION_SUFFIX
elif config.aggregation_type == _ASMK_STAR:
extension += _ASMK_STAR_EXTENSION_SUFFIX
else:
raise ValueError('Invalid aggregation type: %d' %
config.aggregation_type)
num_images = len(image_list)
aggregated_descriptors = []
visual_words = []
print('Starting to collect descriptors for %d images...' % num_images)
start = time.clock()
for i in range(num_images):
if i > 0 and i % _STATUS_CHECK_LOAD_ITERATIONS == 0:
elapsed = (time.clock() - start)
print('Reading descriptors for image %d out of %d, last %d '
'images took %f seconds' %
(i, num_images, _STATUS_CHECK_LOAD_ITERATIONS, elapsed))
start = time.clock()
descriptors_filename = image_list[i] + extension
descriptors_fullpath = os.path.join(input_dir, descriptors_filename)
if config.aggregation_type == _VLAD:
aggregated_descriptors.append(
datum_io.ReadFromFile(descriptors_fullpath))
else:
d, v = datum_io.ReadPairFromFile(descriptors_fullpath)
if config.aggregation_type == _ASMK_STAR:
d = d.astype('uint8')
aggregated_descriptors.append(d)
visual_words.append(v)
return aggregated_descriptors, visual_words
def MakeExtractor(config):
"""Creates a function to extract global and/or local features from an image.
Args:
config: DelfConfig proto containing the model configuration.
Returns:
Function that receives an image and returns features.
Raises:
ValueError: if config is invalid.
"""
# Assert the configuration
if config.use_global_features and hasattr(
config, 'is_tf2_exported') and config.is_tf2_exported:
raise ValueError('use_global_features is incompatible with is_tf2_exported')
# Load model.
model = tf.saved_model.load(config.model_path)
# Input/output end-points/tensors.
feeds = ['input_image:0', 'input_scales:0']
fetches = []
image_scales_tensor = tf.convert_to_tensor(list(config.image_scales))
# Custom configuration needed when local features are used.
if config.use_local_features:
# Extra input/output end-points/tensors.
feeds.append('input_abs_thres:0')
feeds.append('input_max_feature_num:0')
fetches.append('boxes:0')
fetches.append('features:0')
fetches.append('scales:0')
fetches.append('scores:0')
score_threshold_tensor = tf.constant(
config.delf_local_config.score_threshold)
max_feature_num_tensor = tf.constant(
config.delf_local_config.max_feature_num)
# If using PCA, pre-load required parameters.
local_pca_parameters = {}
if config.delf_local_config.use_pca:
local_pca_parameters['mean'] = tf.constant(
datum_io.ReadFromFile(
config.delf_local_config.pca_parameters.mean_path),
dtype=tf.float32)
local_pca_parameters['matrix'] = tf.constant(
datum_io.ReadFromFile(
config.delf_local_config.pca_parameters.projection_matrix_path),
dtype=tf.float32)
local_pca_parameters[
'dim'] = config.delf_local_config.pca_parameters.pca_dim
local_pca_parameters['use_whitening'] = (
config.delf_local_config.pca_parameters.use_whitening)
if config.delf_local_config.pca_parameters.use_whitening:
local_pca_parameters['variances'] = tf.squeeze(
tf.constant(
datum_io.ReadFromFile(
config.delf_local_config.pca_parameters.pca_variances_path),
dtype=tf.float32))
else:
local_pca_parameters['variances'] = None
# Custom configuration needed when global features are used.
if config.use_global_features:
# Extra output end-point.
fetches.append('global_descriptors:0')
# If using PCA, pre-load required parameters.
global_pca_parameters = {}
if config.delf_global_config.use_pca:
global_pca_parameters['mean'] = tf.constant(
datum_io.ReadFromFile(
config.delf_global_config.pca_parameters.mean_path),
dtype=tf.float32)
global_pca_parameters['matrix'] = tf.constant(
datum_io.ReadFromFile(
config.delf_global_config.pca_parameters.projection_matrix_path),
dtype=tf.float32)
global_pca_parameters[
'dim'] = config.delf_global_config.pca_parameters.pca_dim
global_pca_parameters['use_whitening'] = (
config.delf_global_config.pca_parameters.use_whitening)
if config.delf_global_config.pca_parameters.use_whitening:
global_pca_parameters['variances'] = tf.squeeze(
tf.constant(
datum_io.ReadFromFile(config.delf_global_config.pca_parameters
.pca_variances_path),
dtype=tf.float32))
else:
global_pca_parameters['variances'] = None
if not hasattr(config, 'is_tf2_exported') or not config.is_tf2_exported:
model = model.prune(feeds=feeds, fetches=fetches)
def ExtractorFn(image, resize_factor=1.0):
"""Receives an image and returns DELF global and/or local features.
If image is too small, returns empty features.
Args:
image: Uint8 array with shape (height, width, 3) containing the RGB image.
resize_factor: Optional float resize factor for the input image. If given,
the maximum and minimum allowed image sizes in the config are scaled by
this factor.
Returns:
extracted_features: A dict containing the extracted global descriptors
(key 'global_descriptor' mapping to a [D] float array), and/or local
features (key 'local_features' mapping to a dict with keys 'locations',
'descriptors', 'scales', 'attention').
"""
resized_image, scale_factors = utils.ResizeImage(
image, config, resize_factor=resize_factor)
# If the image is too small, returns empty features.
if resized_image.shape[0] < _MIN_HEIGHT or resized_image.shape[
1] < _MIN_WIDTH:
extracted_features = {'global_descriptor': np.array([])}
if config.use_local_features:
extracted_features.update({
'local_features': {
'locations': np.array([]),
'descriptors': np.array([]),
'scales': np.array([]),
'attention': np.array([]),
}
})
return extracted_features
# Input tensors.
image_tensor = tf.convert_to_tensor(resized_image)
# Extracted features.
extracted_features = {}
output = None
if config.use_local_features:
if hasattr(config, 'is_tf2_exported') and config.is_tf2_exported:
predict = model.signatures['serving_default']
output_dict = predict(
input_image=image_tensor,
input_scales=image_scales_tensor,
input_max_feature_num=max_feature_num_tensor,
input_abs_thres=score_threshold_tensor)
output = [
output_dict['boxes'], output_dict['features'],
output_dict['scales'], output_dict['scores']
]
else:
output = model(image_tensor, image_scales_tensor,
score_threshold_tensor, max_feature_num_tensor)
else:
output = model(image_tensor, image_scales_tensor)
# Post-process extracted features: normalize, PCA (optional), pooling.
if config.use_global_features:
raw_global_descriptors = output[-1]
if config.delf_global_config.image_scales_ind:
raw_global_descriptors_selected_scales = tf.gather(
raw_global_descriptors,
list(config.delf_global_config.image_scales_ind))
else:
raw_global_descriptors_selected_scales = raw_global_descriptors
global_descriptors_per_scale = feature_extractor.PostProcessDescriptors(
raw_global_descriptors_selected_scales,
config.delf_global_config.use_pca, global_pca_parameters)
unnormalized_global_descriptor = tf.reduce_sum(
global_descriptors_per_scale, axis=0, name='sum_pooling')
global_descriptor = tf.nn.l2_normalize(
unnormalized_global_descriptor, axis=0, name='final_l2_normalization')
extracted_features.update({
'global_descriptor': global_descriptor.numpy(),
})
if config.use_local_features:
boxes = output[0]
raw_local_descriptors = output[1]
feature_scales = output[2]
attention_with_extra_dim = output[3]
attention = tf.reshape(attention_with_extra_dim,
[tf.shape(attention_with_extra_dim)[0]])
locations, local_descriptors = (
feature_extractor.DelfFeaturePostProcessing(
boxes, raw_local_descriptors, config.delf_local_config.use_pca,
local_pca_parameters))
locations /= scale_factors
extracted_features.update({
'local_features': {
'locations': locations.numpy(),
'descriptors': local_descriptors.numpy(),
'scales': feature_scales.numpy(),
'attention': attention.numpy(),
}
})
return extracted_features
return ExtractorFn
