def DelfFeaturePostProcessing(boxes, descriptors, config): """Extract DELF features from input image. Args: boxes: [N, 4] float tensor which denotes the selected receptive box. N is the number of final feature points which pass through keypoint selection and NMS steps. descriptors: [N, input_dim] float tensor. config: DelfConfig proto with DELF extraction options. Returns: locations: [N, 2] float tensor which denotes the selected keypoint locations. final_descriptors: [N, output_dim] float tensor with DELF descriptors after normalization and (possibly) PCA/whitening. """ # Get center of descriptor boxes, corresponding to feature locations. locations = CalculateKeypointCenters(boxes) # Post-process descriptors: L2-normalize, and if desired apply PCA (followed # by L2-normalization). with tf.variable_scope('postprocess'): final_descriptors = tf.nn.l2_normalize(descriptors, dim=1, name='l2_normalization') if config.delf_local_config.use_pca: # Load PCA parameters. pca_mean = tf.constant(datum_io.ReadFromFile( config.delf_local_config.pca_parameters.mean_path), dtype=tf.float32) pca_matrix = tf.constant( datum_io.ReadFromFile(config.delf_local_config.pca_parameters. projection_matrix_path), dtype=tf.float32) pca_dim = config.delf_local_config.pca_parameters.pca_dim pca_variances = None if config.delf_local_config.pca_parameters.use_whitening: pca_variances = tf.constant( datum_io.ReadFromFile(config.delf_local_config. pca_parameters.pca_variances_path), dtype=tf.float32) # Apply PCA, and whitening if desired. final_descriptors = ApplyPcaAndWhitening( final_descriptors, pca_matrix, pca_mean, pca_dim, config.delf_local_config.pca_parameters.use_whitening, pca_variances) # Re-normalize. final_descriptors = tf.nn.l2_normalize(final_descriptors, dim=1, name='pca_l2_normalization') return locations, final_descriptors
def _ReadDelgGlobalDescriptors(input_dir, image_list): """Reads DELG global features. Args: input_dir: Directory where features are located. image_list: List of image names for which to load features. Returns: global_descriptors: NumPy array of shape (len(image_list), D), where D corresponds to the global descriptor dimensionality. """ num_images = len(image_list) global_descriptors = [] print('Starting to collect global descriptors for %d images...' % num_images) start = time.time() for i in range(num_images): if i > 0 and i % _STATUS_CHECK_LOAD_ITERATIONS == 0: elapsed = (time.time() - start) print('Reading global descriptors for image %d out of %d, last %d ' 'images took %f seconds' % (i, num_images, _STATUS_CHECK_LOAD_ITERATIONS, elapsed)) start = time.time() descriptor_filename = image_list[i] + _DELG_GLOBAL_EXTENSION descriptor_fullpath = os.path.join(input_dir, descriptor_filename) global_descriptors.append(datum_io.ReadFromFile(descriptor_fullpath)) return np.array(global_descriptors)
def testWriteAndReadToFile(self): data = np.array([[[-1.0, 125.0, -2.5], [14.5, 3.5, 0.0]], [[20.0, 0.0, 30.0], [25.5, 36.0, 42.0]]]) filename = os.path.join(FLAGS.test_tmpdir, 'test.datum') datum_io.WriteToFile(data, filename) data_read = datum_io.ReadFromFile(filename) self.assertAllEqual(data_read, data)
def DelfFeaturePostProcessing(boxes, descriptors, config): """ compute centers of receptive fields; apply PCA + whitening to features """ locations = CalculateKeypointCenters(boxes) with tf.variable_scope('postprocess'): final_descriptors = tf.nn.l2_normalize(descriptors, axis=1, name='l2_normalization') if config.delf_local_config.use_pca: pca_dim = config.delf_local_config.pca_parameters.pca_dim mean_path = os.path.expanduser( config.delf_local_config.pca_parameters.mean_path) pca_mean = tf.constant(datum_io.ReadFromFile(mean_path), dtype=tf.float32) projection_matrix_path = os.path.expanduser( config.delf_local_config.pca_parameters.projection_matrix_path) pca_matrix = tf.constant( datum_io.ReadFromFile(projection_matrix_path), dtype=tf.float32) if config.delf_local_config.pca_parameters.use_whitening: pca_variances_path = os.path.expanduser( config.delf_local_config.pca_parameters.pca_variances_path) pca_variances = tf.constant( datum_io.ReadFromFile(pca_variances_path), dtype=tf.float32) else: pca_variances = None final_descriptors = ApplyPcaAndWhitening( final_descriptors, pca_matrix, pca_mean, pca_dim, config.delf_local_config.pca_parameters.use_whitening, pca_variances) final_descriptors = tf.nn.l2_normalize(final_descriptors, axis=1, name='pca_l2_normalization') return locations, final_descriptors
def PostProcessDescriptors(descriptors, use_pca, pca_parameters): """Post-process descriptors. Args: descriptors: [N, input_dim] float tensor. use_pca: Whether to use PCA. pca_parameters: DelfPcaParameters proto. Returns: final_descriptors: [N, output_dim] float tensor with descriptors after normalization and (possibly) PCA/whitening. """ # L2-normalize, and if desired apply PCA (followed by L2-normalization). with tf.compat.v1.variable_scope('postprocess'): final_descriptors = tf.nn.l2_normalize( descriptors, axis=1, name='l2_normalization') if use_pca: # Load PCA parameters. pca_mean = tf.constant( datum_io.ReadFromFile(pca_parameters.mean_path), dtype=tf.float32) pca_matrix = tf.constant( datum_io.ReadFromFile(pca_parameters.projection_matrix_path), dtype=tf.float32) pca_dim = pca_parameters.pca_dim pca_variances = None if pca_parameters.use_whitening: pca_variances = tf.squeeze( tf.constant( datum_io.ReadFromFile(pca_parameters.pca_variances_path), dtype=tf.float32)) # Apply PCA, and whitening if desired. final_descriptors = ApplyPcaAndWhitening(final_descriptors, pca_matrix, pca_mean, pca_dim, pca_parameters.use_whitening, pca_variances) # Re-normalize. final_descriptors = tf.nn.l2_normalize( final_descriptors, axis=1, name='pca_l2_normalization') return final_descriptors
def _ReadAggregatedDescriptors(input_dir, image_list, config): """Reads aggregated descriptors. Args: input_dir: Directory where aggregated descriptors are located. image_list: List of image names for which to load descriptors. config: AggregationConfig used for images. Returns: aggregated_descriptors: List containing #images items, each a 1D NumPy array. visual_words: If using VLAD aggregation, returns an empty list. Otherwise, returns a list containing #images items, each a 1D NumPy array. """ # Compose extension of aggregated descriptors. extension = '.' if config.use_regional_aggregation: extension += 'r' if config.aggregation_type == _VLAD: extension += _VLAD_EXTENSION_SUFFIX elif config.aggregation_type == _ASMK: extension += _ASMK_EXTENSION_SUFFIX elif config.aggregation_type == _ASMK_STAR: extension += _ASMK_STAR_EXTENSION_SUFFIX else: raise ValueError('Invalid aggregation type: %d' % config.aggregation_type) num_images = len(image_list) aggregated_descriptors = [] visual_words = [] print('Starting to collect descriptors for %d images...' % num_images) start = time.clock() for i in range(num_images): if i > 0 and i % _STATUS_CHECK_LOAD_ITERATIONS == 0: elapsed = (time.clock() - start) print('Reading descriptors for image %d out of %d, last %d ' 'images took %f seconds' % (i, num_images, _STATUS_CHECK_LOAD_ITERATIONS, elapsed)) start = time.clock() descriptors_filename = image_list[i] + extension descriptors_fullpath = os.path.join(input_dir, descriptors_filename) if config.aggregation_type == _VLAD: aggregated_descriptors.append( datum_io.ReadFromFile(descriptors_fullpath)) else: d, v = datum_io.ReadPairFromFile(descriptors_fullpath) if config.aggregation_type == _ASMK_STAR: d = d.astype('uint8') aggregated_descriptors.append(d) visual_words.append(v) return aggregated_descriptors, visual_words
def MakeExtractor(config): """Creates a function to extract global and/or local features from an image. Args: config: DelfConfig proto containing the model configuration. Returns: Function that receives an image and returns features. Raises: ValueError: if config is invalid. """ # Assert the configuration if config.use_global_features and hasattr( config, 'is_tf2_exported') and config.is_tf2_exported: raise ValueError('use_global_features is incompatible with is_tf2_exported') # Load model. model = tf.saved_model.load(config.model_path) # Input/output end-points/tensors. feeds = ['input_image:0', 'input_scales:0'] fetches = [] image_scales_tensor = tf.convert_to_tensor(list(config.image_scales)) # Custom configuration needed when local features are used. if config.use_local_features: # Extra input/output end-points/tensors. feeds.append('input_abs_thres:0') feeds.append('input_max_feature_num:0') fetches.append('boxes:0') fetches.append('features:0') fetches.append('scales:0') fetches.append('scores:0') score_threshold_tensor = tf.constant( config.delf_local_config.score_threshold) max_feature_num_tensor = tf.constant( config.delf_local_config.max_feature_num) # If using PCA, pre-load required parameters. local_pca_parameters = {} if config.delf_local_config.use_pca: local_pca_parameters['mean'] = tf.constant( datum_io.ReadFromFile( config.delf_local_config.pca_parameters.mean_path), dtype=tf.float32) local_pca_parameters['matrix'] = tf.constant( datum_io.ReadFromFile( config.delf_local_config.pca_parameters.projection_matrix_path), dtype=tf.float32) local_pca_parameters[ 'dim'] = config.delf_local_config.pca_parameters.pca_dim local_pca_parameters['use_whitening'] = ( config.delf_local_config.pca_parameters.use_whitening) if config.delf_local_config.pca_parameters.use_whitening: local_pca_parameters['variances'] = tf.squeeze( tf.constant( datum_io.ReadFromFile( config.delf_local_config.pca_parameters.pca_variances_path), dtype=tf.float32)) else: local_pca_parameters['variances'] = None # Custom configuration needed when global features are used. if config.use_global_features: # Extra output end-point. fetches.append('global_descriptors:0') # If using PCA, pre-load required parameters. global_pca_parameters = {} if config.delf_global_config.use_pca: global_pca_parameters['mean'] = tf.constant( datum_io.ReadFromFile( config.delf_global_config.pca_parameters.mean_path), dtype=tf.float32) global_pca_parameters['matrix'] = tf.constant( datum_io.ReadFromFile( config.delf_global_config.pca_parameters.projection_matrix_path), dtype=tf.float32) global_pca_parameters[ 'dim'] = config.delf_global_config.pca_parameters.pca_dim global_pca_parameters['use_whitening'] = ( config.delf_global_config.pca_parameters.use_whitening) if config.delf_global_config.pca_parameters.use_whitening: global_pca_parameters['variances'] = tf.squeeze( tf.constant( datum_io.ReadFromFile(config.delf_global_config.pca_parameters .pca_variances_path), dtype=tf.float32)) else: global_pca_parameters['variances'] = None if not hasattr(config, 'is_tf2_exported') or not config.is_tf2_exported: model = model.prune(feeds=feeds, fetches=fetches) def ExtractorFn(image, resize_factor=1.0): """Receives an image and returns DELF global and/or local features. If image is too small, returns empty features. Args: image: Uint8 array with shape (height, width, 3) containing the RGB image. resize_factor: Optional float resize factor for the input image. If given, the maximum and minimum allowed image sizes in the config are scaled by this factor. Returns: extracted_features: A dict containing the extracted global descriptors (key 'global_descriptor' mapping to a [D] float array), and/or local features (key 'local_features' mapping to a dict with keys 'locations', 'descriptors', 'scales', 'attention'). """ resized_image, scale_factors = utils.ResizeImage( image, config, resize_factor=resize_factor) # If the image is too small, returns empty features. if resized_image.shape[0] < _MIN_HEIGHT or resized_image.shape[ 1] < _MIN_WIDTH: extracted_features = {'global_descriptor': np.array([])} if config.use_local_features: extracted_features.update({ 'local_features': { 'locations': np.array([]), 'descriptors': np.array([]), 'scales': np.array([]), 'attention': np.array([]), } }) return extracted_features # Input tensors. image_tensor = tf.convert_to_tensor(resized_image) # Extracted features. extracted_features = {} output = None if config.use_local_features: if hasattr(config, 'is_tf2_exported') and config.is_tf2_exported: predict = model.signatures['serving_default'] output_dict = predict( input_image=image_tensor, input_scales=image_scales_tensor, input_max_feature_num=max_feature_num_tensor, input_abs_thres=score_threshold_tensor) output = [ output_dict['boxes'], output_dict['features'], output_dict['scales'], output_dict['scores'] ] else: output = model(image_tensor, image_scales_tensor, score_threshold_tensor, max_feature_num_tensor) else: output = model(image_tensor, image_scales_tensor) # Post-process extracted features: normalize, PCA (optional), pooling. if config.use_global_features: raw_global_descriptors = output[-1] if config.delf_global_config.image_scales_ind: raw_global_descriptors_selected_scales = tf.gather( raw_global_descriptors, list(config.delf_global_config.image_scales_ind)) else: raw_global_descriptors_selected_scales = raw_global_descriptors global_descriptors_per_scale = feature_extractor.PostProcessDescriptors( raw_global_descriptors_selected_scales, config.delf_global_config.use_pca, global_pca_parameters) unnormalized_global_descriptor = tf.reduce_sum( global_descriptors_per_scale, axis=0, name='sum_pooling') global_descriptor = tf.nn.l2_normalize( unnormalized_global_descriptor, axis=0, name='final_l2_normalization') extracted_features.update({ 'global_descriptor': global_descriptor.numpy(), }) if config.use_local_features: boxes = output[0] raw_local_descriptors = output[1] feature_scales = output[2] attention_with_extra_dim = output[3] attention = tf.reshape(attention_with_extra_dim, [tf.shape(attention_with_extra_dim)[0]]) locations, local_descriptors = ( feature_extractor.DelfFeaturePostProcessing( boxes, raw_local_descriptors, config.delf_local_config.use_pca, local_pca_parameters)) locations /= scale_factors extracted_features.update({ 'local_features': { 'locations': locations.numpy(), 'descriptors': local_descriptors.numpy(), 'scales': feature_scales.numpy(), 'attention': attention.numpy(), } }) return extracted_features return ExtractorFn