def test_search_step(self):
sqltable_in_ds, sqltable_out_ds = get_two_sqlite_data_stores(
SQLTabelDataStoreComputeDescriptorsTestCase.db_path,
SQLTabelDataStoreComputeDescriptorsTestCase.table_name_in,
SQLTabelDataStoreComputeDescriptorsTestCase.table_name_out,
"ndarray")
with sqltable_in_ds:
items = sqltable_in_ds.get_items_sorted_by_ids()
count_ = sqltable_in_ds.get_count()
items = aggregate_iterable(items, count_)
ids = np.arange(1, 11)
searcher_ = exhaustive_searcher.ExhaustiveSearcher(items, ids)
Q = np.arange(8).reshape((1, -1))
np_ds = numpy_datastore.NumpyDataStore(Q)
steps.search_step(np_ds,
searcher_,
10,
sqltable_out_ds,
force=True)
truth_nearest_ids = np.arange(1, 11)
with sqltable_out_ds:
items = sqltable_out_ds.get_items_sorted_by_ids()
first_item = next(items)
self.assertTrue(np.array_equal(truth_nearest_ids, first_item))
def get_items_sorted_by_ids(self, ids_sorted: np.ndarray = None):
with ExitStack() as stack:
if hasattr(self.stream_data_store, '__enter__'):
stack.enter_context(self.stream_data_store)
if not self.stream_data_store.is_stream_data_store():
items_sorted_by_ids = self.stream_data_store.get_items_sorted_by_ids(
ids_sorted)
else:
ids_sorted_stream = None
if ids_sorted is not None:
count_ = ids_sorted.shape[0]
ids_sorted = ids_sorted.ravel()
ids_sorted_stream = iter(ids_sorted)
else:
count_ = self.stream_data_store.get_count()
items_sorted_by_ids_stream = self.stream_data_store.get_items_sorted_by_ids(
ids_sorted_stream)
items_sorted_by_ids_ndarray = aggregate_iterable(
items_sorted_by_ids_stream,
detect_final_shape_by_first_elem=self.
detect_final_shape_by_first_elem,
final_shape=self.shape,
n_elements=count_)
items_sorted_by_ids = items_sorted_by_ids_ndarray
if self.shape is not None:
items_sorted_by_ids = items_sorted_by_ids.reshape(self.shape)
if self.slice_get is not None:
items_sorted_by_ids = items_sorted_by_ids[self.slice_get]
return items_sorted_by_ids
def chunkify(self, chunk_size=-1):
if not self.is_stream_chunkified:
chunk_sizes = self.chunk_sizes_(chunk_size)
it = iter(self.data_stream)
chunks_stream = (aggregate_iterable(it, detect_final_shape_by_first_elem=True,
n_elements=chunk_size) for chunk_size in chunk_sizes)
self.data_stream = chunks_stream
self.is_stream_chunkified = True
def wrapper(*args):
new_args = []
for arg in args:
new_arg = arg
if not isinstance(arg, np.ndarray) and isinstance(
arg, collections.Iterable):
aggregated = aggregate_iterable(arg, )
new_arg = aggregated
new_args.append(new_arg)
return func(*new_args)
def get_ids_sorted(self):
with ExitStack() as stack:
if hasattr(self.stream_data_store, '__enter__'):
stack.enter_context(self.stream_data_store)
if not self.stream_data_store.is_stream_data_store():
ids_sorted_ndarray = self.stream_data_store.get_ids_sorted()
else:
ids_sorted = self.stream_data_store.get_ids_sorted()
count_ = self.stream_data_store.get_count()
ids_sorted_ndarray = aggregate_iterable(ids_sorted,
n_elements=count_)
return ids_sorted_ndarray
def save_items_sorted_by_ids(self, items_sorted_by_ids: np.ndarray, ids_sorted: np.ndarray = None):
if items_sorted_by_ids is None:
return
if ids_sorted is None:
if isinstance(items_sorted_by_ids, list):
items_sorted_by_ids = np.asarray(items_sorted_by_ids)
ids_sorted = np.arange(1, len(items_sorted_by_ids) + 1)
elif not isinstance(items_sorted_by_ids, np.ndarray):
items_sorted_by_ids, items_sorted_by_ids_copy_ = itertools.tee(items_sorted_by_ids)
items_len = sum(1 for _ in items_sorted_by_ids_copy_)
ids_sorted = np.arange(1, items_len + 1)
else:
ids_sorted = np.arange(1, len(items_sorted_by_ids) + 1)
elif not isinstance(ids_sorted, np.ndarray):
ids_sorted = aggregate_iterable(ids_sorted)
if not isinstance(items_sorted_by_ids, np.ndarray):
items_sorted_by_ids = aggregate_iterable(items_sorted_by_ids, detect_final_shape_by_first_elem=True,
n_elements=len(ids_sorted))
self.items_sorted_by_ids = items_sorted_by_ids
self.ids_sorted = ids_sorted
def test_quantize_shape_only(self):
sqltable_in_ds, sqltable_out_ds = get_two_sqlite_data_stores(
SQLTabelDataStoreComputeDescriptorsTestCase.db_path,
SQLTabelDataStoreComputeDescriptorsTestCase.table_name_in,
SQLTabelDataStoreComputeDescriptorsTestCase.table_name_out,
"ndarray")
quantizer = pq_quantizer.PQQuantizer(n_clusters=4, n_quantizers=2)
steps.quantize_step(sqltable_in_ds,
quantizer,
sqltable_out_ds,
force=True)
with sqltable_out_ds:
cluster_centers = sqltable_out_ds.get_items_sorted_by_ids()
cluster_centers_ndarray = aggregate_iterable(
cluster_centers, detect_final_shape_by_first_elem=True)
truth_shape = (2, 4, 4)
self.assertEquals(cluster_centers_ndarray.shape, truth_shape)
def aggregate_items(foreignid_itemsiterable):
itemsiterable = foreignid_itemsiterable[1]
itemsiterable = map(lambda foreignid_item: foreignid_item[1], itemsiterable)
aggregated_items = ai.aggregate_iterable(itemsiterable, detect_final_shape_by_first_elem=True)
return (foreignid_itemsiterable[0], aggregated_items)