def test_remove_from_index(self) -> None: # Test that actual removal occurs. i = LinearHashIndex() i.index = {0, 1, 2} # noinspection PyTypeChecker i.remove_from_index([[0, 0], [1, 0]]) self.assertSetEqual(i.index, {1})
def test_remove_from_index_single_not_in_index(self) -> None: # Test attempting to remove single hash not in the index. i = LinearHashIndex() i.index = {0, 1, 2} self.assertRaises( KeyError, i.remove_from_index, [[1, 0, 0]] # 4 ) self.assertSetEqual(i.index, {0, 1, 2})
def test_remove_from_index_one_of_many_not_in_index(self) -> None: # Test attempting to remove hashes where one of them is not in the # index. i = LinearHashIndex() i.index = {0, 1, 2} self.assertRaises( KeyError, i.remove_from_index, [ [0, 0], # 0 [0, 1], # 1 [1, 1] ] # 3 ) # Check that the index has not been modified. self.assertSetEqual(i.index, {0, 1, 2})
def _nn( self, d: DescriptorElement, n: int = 1 ) -> Tuple[Tuple[DescriptorElement, ...], Tuple[float, ...]]: """ Internal method to be implemented by sub-classes to return the nearest `N` neighbors to the given descriptor element. When this internal method is called, we have already checked that there is a vector in ``d`` and our index is not empty. :param d: Descriptor element to compute the neighbors of. :param n: Number of nearest neighbors to find. :return: Tuple of nearest N DescriptorElement instances, and a tuple of the distance values to those neighbors. """ LOG.debug("generating hash for descriptor") d_v = d.vector() d_h = self.lsh_functor.get_hash(d_v) def comp_descr_dist(d2_v: numpy.ndarray) -> float: return self._distance_function(d_v, d2_v) with self._model_lock: LOG.debug("getting near hashes") hi = self.hash_index if hi is None: # Make on-the-fly linear index hi = LinearHashIndex() # not calling ``build_index`` because we already have the int # hashes. hi.index = set(cast(Iterator[int], self.hash2uuids_kvstore.keys())) near_hashes, _ = hi.nn(d_h, n) LOG.debug("getting UUIDs of descriptors for nearby hashes") neighbor_uuids: List[Hashable] = [] for h_int in map(bit_vector_to_int_large, near_hashes): # If descriptor hash not in our map, we effectively skip it. # Get set of descriptor UUIDs for a hash code. near_uuids: Set[Hashable] = self.hash2uuids_kvstore.get(h_int, set()) # Accumulate matching descriptor UUIDs to a list. neighbor_uuids.extend(near_uuids) LOG.debug("-- matched %d UUIDs", len(neighbor_uuids)) LOG.debug("getting descriptors for neighbor_uuids") neighbors = \ list(self.descriptor_set.get_many_descriptors(neighbor_uuids)) # Done with model parts at this point, so releasing lock. LOG.debug(f"ordering descriptors via distance method {self.distance_method}") LOG.debug('-- getting element vectors') neighbor_vectors = numpy.asarray(list( parallel_map(lambda d_: d_.vector(), neighbors) )) LOG.debug('-- calculating distances') distances = list(map(comp_descr_dist, neighbor_vectors)) LOG.debug('-- ordering') ordered = sorted(zip(neighbors, distances), key=lambda p: p[1]) LOG.debug(f'-- slicing top n={n}') r_descrs: Tuple[DescriptorElement, ...] r_dists: Tuple[float, ...] r_descrs, r_dists = zip(*(ordered[:n])) return r_descrs, r_dists