def testComputeRasmkWorks(self):
# Construct inputs.
# 4 2-D features: 3 in first region, 1 in second region.
features = np.array([[1.0, 0.0], [-1.0, 0.0], [1.0, 2.0], [0.0, 2.0]],
dtype=float)
num_features_per_region = np.array([3, 1])
config = aggregation_config_pb2.AggregationConfig()
config.codebook_size = 5
config.feature_dimensionality = 2
config.aggregation_type = aggregation_config_pb2.AggregationConfig.ASMK
config.codebook_path = self._codebook_path
config.num_assignments = 1
config.use_regional_aggregation = True
# Run tested function.
with tf.Graph().as_default() as g, self.session(graph=g) as sess:
extractor = feature_aggregation_extractor.ExtractAggregatedRepresentation(
sess, config)
rasmk, visual_words = extractor.Extract(features, num_features_per_region)
# Define expected results.
exp_rasmk = [-0.707107, 0.707107, 0.361261, 0.932465]
exp_visual_words = [3, 4]
# Compare actual and expected results.
self.assertAllClose(rasmk, exp_rasmk)
self.assertAllEqual(visual_words, exp_visual_words)
def testComputeNormalizedVladWithBatchingWorks(self):
# Construct inputs.
# 3 2-D features.
features = np.array([[1.0, 0.0], [-1.0, 0.0], [1.0, 2.0]], dtype=float)
config = aggregation_config_pb2.AggregationConfig()
config.codebook_size = 5
config.feature_dimensionality = 2
config.aggregation_type = aggregation_config_pb2.AggregationConfig.VLAD
config.use_l2_normalization = True
config.codebook_path = self._codebook_path
config.num_assignments = 1
config.feature_batch_size = 2
# Run tested function.
extractor = feature_aggregation_extractor.ExtractAggregatedRepresentation(
config)
vlad, extra_output = extractor.Extract(features)
# Define expected results.
exp_vlad = [
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -0.316228, 0.316228, 0.632456, 0.632456
]
exp_extra_output = -1
# Compare actual and expected results.
self.assertAllClose(vlad, exp_vlad)
self.assertAllEqual(extra_output, exp_extra_output)
def testComputeRasmkStarWorks(self):
# Construct inputs.
# 4 2-D features: 3 in first region, 1 in second region.
features = np.array([[1.0, 0.0], [-1.0, 0.0], [1.0, 2.0], [0.0, 2.0]],
dtype=float)
num_features_per_region = np.array([3, 1])
config = aggregation_config_pb2.AggregationConfig()
config.codebook_size = 5
config.feature_dimensionality = 2
config.aggregation_type = aggregation_config_pb2.AggregationConfig.ASMK_STAR
config.codebook_path = self._codebook_path
config.num_assignments = 1
config.use_regional_aggregation = True
# Run tested function.
extractor = feature_aggregation_extractor.ExtractAggregatedRepresentation(
config)
rasmk_star, visual_words = extractor.Extract(features,
num_features_per_region)
# Define expected results.
exp_rasmk_star = [64, 192]
exp_visual_words = [3, 4]
# Compare actual and expected results.
self.assertAllEqual(rasmk_star, exp_rasmk_star)
self.assertAllEqual(visual_words, exp_visual_words)
def testComputeNormalizedRvladSomeEmptyRegionsWorks(self):
# Construct inputs.
# 4 2-D features: 0 in first region, 3 in second region, 0 in third region,
# 1 in fourth region.
features = np.array([[1.0, 0.0], [-1.0, 0.0], [1.0, 2.0], [0.0, 2.0]],
dtype=float)
num_features_per_region = np.array([0, 3, 0, 1])
config = aggregation_config_pb2.AggregationConfig()
config.codebook_size = 5
config.feature_dimensionality = 2
config.aggregation_type = aggregation_config_pb2.AggregationConfig.VLAD
config.use_l2_normalization = True
config.codebook_path = self._codebook_path
config.num_assignments = 1
config.use_regional_aggregation = True
# Run tested function.
extractor = feature_aggregation_extractor.ExtractAggregatedRepresentation(
config)
rvlad, extra_output = extractor.Extract(features, num_features_per_region)
# Define expected results.
exp_rvlad = [
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -0.175011, 0.175011, 0.350021, 0.903453
]
exp_extra_output = -1
# Compare actual and expected results.
self.assertAllClose(rvlad, exp_rvlad)
self.assertAllEqual(extra_output, exp_extra_output)
def testComputeRvladEmptyRegionsWorks(self):
# Construct inputs.
# Empty feature array.
features = np.array([[]])
num_features_per_region = np.array([])
config = aggregation_config_pb2.AggregationConfig()
config.codebook_size = 5
config.feature_dimensionality = 2
config.aggregation_type = aggregation_config_pb2.AggregationConfig.VLAD
config.codebook_path = self._codebook_path
config.use_regional_aggregation = True
# Run tested function.
with tf.Graph().as_default() as g, self.session(graph=g) as sess:
extractor = feature_aggregation_extractor.ExtractAggregatedRepresentation(
sess, config)
rvlad, extra_output = extractor.Extract(features, num_features_per_region)
# Define expected results.
exp_rvlad = np.zeros([10], dtype=float)
exp_extra_output = -1
# Compare actual and expected results.
self.assertAllEqual(rvlad, exp_rvlad)
self.assertAllEqual(extra_output, exp_extra_output)
def testComputeUnnormalizedRvladWorks(self):
# Construct inputs.
# 4 2-D features: 3 in first region, 1 in second region.
features = np.array([[1.0, 0.0], [-1.0, 0.0], [1.0, 2.0], [0.0, 2.0]],
dtype=float)
num_features_per_region = np.array([3, 1])
config = aggregation_config_pb2.AggregationConfig()
config.codebook_size = 5
config.feature_dimensionality = 2
config.aggregation_type = aggregation_config_pb2.AggregationConfig.VLAD
config.use_l2_normalization = False
config.codebook_path = self._codebook_path
config.num_assignments = 1
config.use_regional_aggregation = True
# Run tested function.
with tf.Graph().as_default() as g, self.session(graph=g) as sess:
extractor = feature_aggregation_extractor.ExtractAggregatedRepresentation(
sess, config)
rvlad, extra_output = extractor.Extract(features, num_features_per_region)
# Define expected results.
exp_rvlad = [
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -0.158114, 0.158114, 0.316228, 0.816228
]
exp_extra_output = -1
# Compare actual and expected results.
self.assertAllClose(rvlad, exp_rvlad)
self.assertAllEqual(extra_output, exp_extra_output)
def testComputeUnnormalizedVladMultipleAssignmentWorks(self):
# Construct inputs.
# 3 2-D features.
features = np.array([[1.0, 0.0], [-1.0, 0.0], [1.0, 2.0]], dtype=float)
config = aggregation_config_pb2.AggregationConfig()
config.codebook_size = 5
config.feature_dimensionality = 2
config.aggregation_type = aggregation_config_pb2.AggregationConfig.VLAD
config.use_l2_normalization = False
config.codebook_path = self._codebook_path
config.num_assignments = 3
# Run tested function.
with tf.Graph().as_default() as g, self.session(graph=g) as sess:
extractor = feature_aggregation_extractor.ExtractAggregatedRepresentation(
sess, config)
vlad, extra_output = extractor.Extract(features)
# Define expected results.
exp_vlad = [1.0, 1.0, 0.0, 0.0, 0.0, 2.0, -0.5, 0.5, 0.0, 0.0]
exp_extra_output = -1
# Compare actual and expected results.
self.assertAllEqual(vlad, exp_vlad)
self.assertAllEqual(extra_output, exp_extra_output)
def testComputeUnknownAggregation(self):
# Construct inputs.
config = aggregation_config_pb2.AggregationConfig()
config.codebook_size = 5
config.feature_dimensionality = 2
config.aggregation_type = 0
config.codebook_path = self._codebook_path
config.use_regional_aggregation = True
# Run tested function.
with self.assertRaisesRegex(ValueError, 'Invalid aggregation type'):
feature_aggregation_extractor.ExtractAggregatedRepresentation(
config)
def testComputeVladEmptyFeaturesWorks(self):
# Construct inputs.
# Empty feature array.
features = np.array([[]])
config = aggregation_config_pb2.AggregationConfig()
config.codebook_size = 5
config.feature_dimensionality = 2
config.aggregation_type = aggregation_config_pb2.AggregationConfig.VLAD
config.codebook_path = self._codebook_path
# Run tested function.
extractor = feature_aggregation_extractor.ExtractAggregatedRepresentation(
config)
vlad, extra_output = extractor.Extract(features)
# Define expected results.
exp_vlad = np.zeros([10], dtype=float)
exp_extra_output = -1
# Compare actual and expected results.
self.assertAllEqual(vlad, exp_vlad)
self.assertAllEqual(extra_output, exp_extra_output)
def testComputeAsmkMultipleAssignmentWorks(self):
# Construct inputs.
# 3 2-D features.
features = np.array([[1.0, 0.0], [-1.0, 0.0], [1.0, 2.0]], dtype=float)
config = aggregation_config_pb2.AggregationConfig()
config.codebook_size = 5
config.feature_dimensionality = 2
config.aggregation_type = aggregation_config_pb2.AggregationConfig.ASMK
config.codebook_path = self._codebook_path
config.num_assignments = 3
# Run tested function.
extractor = feature_aggregation_extractor.ExtractAggregatedRepresentation(
config)
asmk, visual_words = extractor.Extract(features)
# Define expected results.
exp_asmk = [0.707107, 0.707107, 0.0, 1.0, -0.707107, 0.707107]
exp_visual_words = [0, 2, 3]
# Compare actual and expected results.
self.assertAllClose(asmk, exp_asmk)
self.assertAllEqual(visual_words, exp_visual_words)
def testComputeAsmkStarWorks(self):
# Construct inputs.
# 3 2-D features.
features = np.array([[1.0, 0.0], [-1.0, 0.0], [1.0, 2.0]], dtype=float)
config = aggregation_config_pb2.AggregationConfig()
config.codebook_size = 5
config.feature_dimensionality = 2
config.aggregation_type = aggregation_config_pb2.AggregationConfig.ASMK_STAR
config.codebook_path = self._codebook_path
config.num_assignments = 1
# Run tested function.
extractor = feature_aggregation_extractor.ExtractAggregatedRepresentation(
config)
asmk_star, visual_words = extractor.Extract(features)
# Define expected results.
exp_asmk_star = [64, 192]
exp_visual_words = [3, 4]
# Compare actual and expected results.
self.assertAllEqual(asmk_star, exp_asmk_star)
self.assertAllEqual(visual_words, exp_visual_words)
def testComputeRvladMisconfiguredFeatures(self):
# Construct inputs.
# 4 2-D features: 3 in first region, 1 in second region.
features = np.array([[1.0, 0.0], [-1.0, 0.0], [1.0, 2.0], [0.0, 2.0]],
dtype=float)
# Misconfigured number of features; there are only 4 features, but
# sum(num_features_per_region) = 5.
num_features_per_region = np.array([3, 2])
config = aggregation_config_pb2.AggregationConfig()
config.codebook_size = 5
config.feature_dimensionality = 2
config.aggregation_type = aggregation_config_pb2.AggregationConfig.VLAD
config.codebook_path = self._codebook_path
config.use_regional_aggregation = True
# Run tested function.
extractor = feature_aggregation_extractor.ExtractAggregatedRepresentation(
config)
with self.assertRaisesRegex(
ValueError,
r'Incorrect arguments: sum\(num_features_per_region\) and '
r'features.shape\[0\] are different'):
extractor.Extract(features, num_features_per_region)
def ExtractAggregatedRepresentationsToFiles(image_names, features_dir,
aggregation_config_path,
mapping_path,
output_aggregation_dir):
"""Extracts aggregated feature representations, saving them to files.
It checks if the aggregated representation for an image already exists,
and skips computation for those.
Args:
image_names: List of image names. These are used to compose input file names
for the feature files, and the output file names for aggregated
representations.
features_dir: Directory where DELF features are located.
aggregation_config_path: Path to AggregationConfig proto text file with
configuration to be used for extraction.
mapping_path: Optional CSV file which maps each .delf file name to the index
image ID and detected box ID. If regional aggregation is performed, this
should be set. Otherwise, this is ignored.
output_aggregation_dir: Directory where aggregation output will be written
to.
Raises:
ValueError: If AggregationConfig is malformed, or `mapping_path` is
missing.
"""
num_images = len(image_names)
# Parse AggregationConfig proto, and select output extension.
config = aggregation_config_pb2.AggregationConfig()
with tf.io.gfile.GFile(aggregation_config_path, 'r') as f:
text_format.Merge(f.read(), config)
output_extension = '.'
if config.use_regional_aggregation:
output_extension += 'r'
if config.aggregation_type == _VLAD:
output_extension += _VLAD_EXTENSION_SUFFIX
elif config.aggregation_type == _ASMK:
output_extension += _ASMK_EXTENSION_SUFFIX
elif config.aggregation_type == _ASMK_STAR:
output_extension += _ASMK_STAR_EXTENSION_SUFFIX
else:
raise ValueError('Invalid aggregation type: %d' %
config.aggregation_type)
# Read index mapping path, if provided.
if mapping_path:
images_to_box_feature_files = _ReadMappingBasenameToBoxNames(
mapping_path, image_names)
# Create output directory if necessary.
if not tf.io.gfile.exists(output_aggregation_dir):
tf.io.gfile.makedirs(output_aggregation_dir)
with tf.compat.v1.Session() as sess:
extractor = feature_aggregation_extractor.ExtractAggregatedRepresentation(
sess, config)
start = time.clock()
for i in range(num_images):
if i == 0:
print('Starting to extract aggregation from images...')
elif i % _STATUS_CHECK_ITERATIONS == 0:
elapsed = (time.clock() - start)
print('Processing image %d out of %d, last %d '
'images took %f seconds' %
(i, num_images, _STATUS_CHECK_ITERATIONS, elapsed))
start = time.clock()
image_name = image_names[i]
# Compose output file name, skip extraction for this image if it already
# exists.
output_aggregation_filename = os.path.join(
output_aggregation_dir, image_name + output_extension)
if tf.io.gfile.exists(output_aggregation_filename):
print('Skipping %s' % image_name)
continue
# Load DELF features.
if config.use_regional_aggregation:
if not mapping_path:
raise ValueError(
'Requested regional aggregation, but mapping_path was not '
'provided')
descriptors_list = []
num_features_per_box = []
for box_feature_file in images_to_box_feature_files[
image_name]:
delf_filename = os.path.join(
features_dir, box_feature_file + _DELF_EXTENSION)
_, _, box_descriptors, _, _ = feature_io.ReadFromFile(
delf_filename)
# If `box_descriptors` is empty, reshape it such that it can be
# concatenated with other descriptors.
if not box_descriptors.shape[0]:
box_descriptors = np.reshape(
box_descriptors,
[0, config.feature_dimensionality])
descriptors_list.append(box_descriptors)
num_features_per_box.append(box_descriptors.shape[0])
descriptors = np.concatenate(descriptors_list)
else:
input_delf_filename = os.path.join(
features_dir, image_name + _DELF_EXTENSION)
_, _, descriptors, _, _ = feature_io.ReadFromFile(
input_delf_filename)
# If `descriptors` is empty, reshape it to avoid extraction failure.
if not descriptors.shape[0]:
descriptors = np.reshape(
descriptors, [0, config.feature_dimensionality])
num_features_per_box = None
# Extract and save aggregation. If using VLAD, only
# `aggregated_descriptors` needs to be saved.
(aggregated_descriptors,
feature_visual_words) = extractor.Extract(descriptors,
num_features_per_box)
if config.aggregation_type == _VLAD:
datum_io.WriteToFile(aggregated_descriptors,
output_aggregation_filename)
else:
datum_io.WritePairToFile(aggregated_descriptors,
feature_visual_words.astype('uint32'),
output_aggregation_filename)
def main(argv):
if len(argv) > 1:
raise RuntimeError('Too many command-line arguments.')
# Read list of images from dataset file.
print('Reading list of images from dataset file...')
query_list, index_list, _ = dataset.ReadDatasetFile(
cmd_args.dataset_file_path)
if cmd_args.use_query_images:
image_list = query_list
else:
image_list = index_list
num_images = len(image_list)
print('done! Found %d images' % num_images)
# Parse AggregationConfig proto, and select output extension.
config = aggregation_config_pb2.AggregationConfig()
with tf.gfile.GFile(cmd_args.aggregation_config_path, 'r') as f:
text_format.Merge(f.read(), config)
output_extension = '.'
if config.use_regional_aggregation:
output_extension += 'r'
if config.aggregation_type == _VLAD:
output_extension += _VLAD_EXTENSION_SUFFIX
elif config.aggregation_type == _ASMK:
output_extension += _ASMK_EXTENSION_SUFFIX
elif config.aggregation_type == _ASMK_STAR:
output_extension += _ASMK_STAR_EXTENSION_SUFFIX
else:
raise ValueError('Invalid aggregation type: %d' %
config.aggregation_type)
# Read index mapping path, if provided.
if cmd_args.index_mapping_path:
images_to_box_feature_files = _ReadMappingBasenameToBoxNames(
cmd_args.index_mapping_path, image_list)
# Create output directory if necessary.
if not os.path.exists(cmd_args.output_aggregation_dir):
os.makedirs(cmd_args.output_aggregation_dir)
with tf.Session() as sess:
extractor = feature_aggregation_extractor.ExtractAggregatedRepresentation(
sess, config)
start = time.clock()
for i in range(num_images):
if i == 0:
print('Starting to extract aggregation from images...')
elif i % _STATUS_CHECK_ITERATIONS == 0:
elapsed = (time.clock() - start)
print('Processing image %d out of %d, last %d '
'images took %f seconds' %
(i, num_images, _STATUS_CHECK_ITERATIONS, elapsed))
start = time.clock()
image_name = image_list[i]
# Compose output file name, skip extraction for this image if it already
# exists.
output_aggregation_filename = os.path.join(
cmd_args.output_aggregation_dir, image_name + output_extension)
if tf.io.gfile.exists(output_aggregation_filename):
print('Skipping %s' % image_name)
continue
# Load DELF features.
if config.use_regional_aggregation:
if not cmd_args.index_mapping_path:
raise ValueError(
'Requested regional aggregation, but index_mapping_path was not '
'provided')
descriptors_list = []
num_features_per_box = []
for box_feature_file in images_to_box_feature_files[
image_name]:
delf_filename = os.path.join(
cmd_args.features_dir,
box_feature_file + _DELF_EXTENSION)
_, _, box_descriptors, _, _ = feature_io.ReadFromFile(
delf_filename)
# If `box_descriptors` is empty, reshape it such that it can be
# concatenated with other descriptors.
if not box_descriptors.shape[0]:
box_descriptors = np.reshape(
box_descriptors,
[0, config.feature_dimensionality])
descriptors_list.append(box_descriptors)
num_features_per_box.append(box_descriptors.shape[0])
descriptors = np.concatenate(descriptors_list)
else:
input_delf_filename = os.path.join(
cmd_args.features_dir, image_name + _DELF_EXTENSION)
_, _, descriptors, _, _ = feature_io.ReadFromFile(
input_delf_filename)
num_features_per_box = None
# Extract and save aggregation. If using VLAD, only
# `aggregated_descriptors` needs to be saved.
(aggregated_descriptors,
feature_visual_words) = extractor.Extract(descriptors,
num_features_per_box)
if config.aggregation_type == _VLAD:
datum_io.WriteToFile(aggregated_descriptors,
output_aggregation_filename)
else:
datum_io.WritePairToFile(aggregated_descriptors,
feature_visual_words.astype('uint32'),
output_aggregation_filename)
