def ExtractorFn(image, resize_factor=1.0):
"""Receives an image and returns DELF global and/or local features.
If image is too small, returns empty features.
Args:
image: Uint8 array with shape (height, width, 3) containing the RGB image.
resize_factor: Optional float resize factor for the input image. If given,
the maximum and minimum allowed image sizes in the config are scaled by
this factor.
Returns:
extracted_features: A dict containing the extracted global descriptors
(key 'global_descriptor' mapping to a [D] float array), and/or local
features (key 'local_features' mapping to a dict with keys 'locations',
'descriptors', 'scales', 'attention').
"""
resized_image, scale_factors = utils.ResizeImage(
image, config, resize_factor=resize_factor)
# If the image is too small, returns empty features.
if resized_image.shape[0] < _MIN_HEIGHT or resized_image.shape[
1] < _MIN_WIDTH:
extracted_features = {'global_descriptor': np.array([])}
if config.use_local_features:
extracted_features.update({
'local_features': {
'locations': np.array([]),
'descriptors': np.array([]),
'scales': np.array([]),
'attention': np.array([]),
}
})
return extracted_features
# Input tensors.
image_tensor = tf.convert_to_tensor(resized_image)
# Extracted features.
extracted_features = {}
output = None
if config.use_local_features:
if hasattr(config, 'is_tf2_exported') and config.is_tf2_exported:
predict = model.signatures['serving_default']
output_dict = predict(
input_image=image_tensor,
input_scales=image_scales_tensor,
input_max_feature_num=max_feature_num_tensor,
input_abs_thres=score_threshold_tensor)
output = [
output_dict['boxes'], output_dict['features'],
output_dict['scales'], output_dict['scores']
]
else:
output = model(image_tensor, image_scales_tensor,
score_threshold_tensor, max_feature_num_tensor)
else:
output = model(image_tensor, image_scales_tensor)
# Post-process extracted features: normalize, PCA (optional), pooling.
if config.use_global_features:
raw_global_descriptors = output[-1]
if config.delf_global_config.image_scales_ind:
raw_global_descriptors_selected_scales = tf.gather(
raw_global_descriptors,
list(config.delf_global_config.image_scales_ind))
else:
raw_global_descriptors_selected_scales = raw_global_descriptors
global_descriptors_per_scale = feature_extractor.PostProcessDescriptors(
raw_global_descriptors_selected_scales,
config.delf_global_config.use_pca, global_pca_parameters)
unnormalized_global_descriptor = tf.reduce_sum(
global_descriptors_per_scale, axis=0, name='sum_pooling')
global_descriptor = tf.nn.l2_normalize(
unnormalized_global_descriptor, axis=0, name='final_l2_normalization')
extracted_features.update({
'global_descriptor': global_descriptor.numpy(),
})
if config.use_local_features:
boxes = output[0]
raw_local_descriptors = output[1]
feature_scales = output[2]
attention_with_extra_dim = output[3]
attention = tf.reshape(attention_with_extra_dim,
[tf.shape(attention_with_extra_dim)[0]])
locations, local_descriptors = (
feature_extractor.DelfFeaturePostProcessing(
boxes, raw_local_descriptors, config.delf_local_config.use_pca,
local_pca_parameters))
locations /= scale_factors
extracted_features.update({
'local_features': {
'locations': locations.numpy(),
'descriptors': local_descriptors.numpy(),
'scales': feature_scales.numpy(),
'attention': attention.numpy(),
}
})
return extracted_features
def MakeExtractor(sess, config, import_scope=None):
"""Creates a function to extract global and/or local features from an image.
Args:
sess: TensorFlow session to use.
config: DelfConfig proto containing the model configuration.
import_scope: Optional scope to use for model.
Returns:
Function that receives an image and returns features.
"""
# Load model.
tf.compat.v1.saved_model.loader.load(
sess, [tf.compat.v1.saved_model.tag_constants.SERVING],
config.model_path,
import_scope=import_scope)
import_scope_prefix = import_scope + '/' if import_scope is not None else ''
# Input tensors.
input_image = sess.graph.get_tensor_by_name('%sinput_image:0' %
import_scope_prefix)
input_image_scales = sess.graph.get_tensor_by_name('%sinput_scales:0' %
import_scope_prefix)
if config.use_local_features:
input_score_threshold = sess.graph.get_tensor_by_name(
'%sinput_abs_thres:0' % import_scope_prefix)
input_max_feature_num = sess.graph.get_tensor_by_name(
'%sinput_max_feature_num:0' % import_scope_prefix)
# Output tensors.
if config.use_global_features:
raw_global_descriptors = sess.graph.get_tensor_by_name(
'%sglobal_descriptors:0' % import_scope_prefix)
if config.use_local_features:
boxes = sess.graph.get_tensor_by_name('%sboxes:0' % import_scope_prefix)
raw_local_descriptors = sess.graph.get_tensor_by_name('%sfeatures:0' %
import_scope_prefix)
feature_scales = sess.graph.get_tensor_by_name('%sscales:0' %
import_scope_prefix)
attention_with_extra_dim = sess.graph.get_tensor_by_name(
'%sscores:0' % import_scope_prefix)
# Post-process extracted features: normalize, PCA (optional), pooling.
if config.use_global_features:
if config.delf_global_config.image_scales_ind:
raw_global_descriptors_selected_scales = tf.gather(
raw_global_descriptors,
list(config.delf_global_config.image_scales_ind))
else:
raw_global_descriptors_selected_scales = raw_global_descriptors
global_descriptors_per_scale = feature_extractor.PostProcessDescriptors(
raw_global_descriptors_selected_scales,
config.delf_global_config.use_pca,
config.delf_global_config.pca_parameters)
unnormalized_global_descriptor = tf.reduce_sum(
global_descriptors_per_scale, axis=0, name='sum_pooling')
global_descriptor = tf.nn.l2_normalize(
unnormalized_global_descriptor, axis=0, name='final_l2_normalization')
if config.use_local_features:
attention = tf.reshape(attention_with_extra_dim,
[tf.shape(attention_with_extra_dim)[0]])
locations, local_descriptors = feature_extractor.DelfFeaturePostProcessing(
boxes, raw_local_descriptors, config)
def ExtractorFn(image, resize_factor=1.0):
"""Receives an image and returns DELF global and/or local features.
If image is too small, returns empty features.
Args:
image: Uint8 array with shape (height, width, 3) containing the RGB image.
resize_factor: Optional float resize factor for the input image. If given,
the maximum and minimum allowed image sizes in the config are scaled by
this factor.
Returns:
extracted_features: A dict containing the extracted global descriptors
(key 'global_descriptor' mapping to a [D] float array), and/or local
features (key 'local_features' mapping to a dict with keys 'locations',
'descriptors', 'scales', 'attention').
"""
resized_image, scale_factors = ResizeImage(
image, config, resize_factor=resize_factor)
# If the image is too small, returns empty features.
if resized_image.shape[0] < _MIN_HEIGHT or resized_image.shape[
1] < _MIN_WIDTH:
extracted_features = {'global_descriptor': np.array([])}
if config.use_local_features:
extracted_features.update({
'local_features': {
'locations': np.array([]),
'descriptors': np.array([]),
'scales': np.array([]),
'attention': np.array([]),
}
})
return extracted_features
feed_dict = {
input_image: resized_image,
input_image_scales: list(config.image_scales),
}
fetches = {}
if config.use_global_features:
fetches.update({
'global_descriptor': global_descriptor,
})
if config.use_local_features:
feed_dict.update({
input_score_threshold: config.delf_local_config.score_threshold,
input_max_feature_num: config.delf_local_config.max_feature_num,
})
fetches.update({
'local_features': {
'locations': locations,
'descriptors': local_descriptors,
'scales': feature_scales,
'attention': attention,
}
})
extracted_features = sess.run(fetches, feed_dict=feed_dict)
# Adjust local feature positions due to rescaling.
if config.use_local_features:
extracted_features['local_features']['locations'] /= scale_factors
return extracted_features
return ExtractorFn