def test_remove():
"""Unit test for method remove."""
index_to_remove = 8
temp_file = FileIO.safe_temp_file(file_name='pupyl.index')
temp_dir = os.path.dirname(temp_file)
with Index(TEST_VECTOR_SIZE, data_dir=temp_dir) as index:
for _ in range(16):
index.append(numpy.random.normal(size=TEST_VECTOR_SIZE))
test_size_before = len(index)
test_value = index[index_to_remove]
with Index(TEST_VECTOR_SIZE, data_dir=temp_dir) as index:
index.remove(index_to_remove)
assert len(index) == test_size_before - 1
numpy.testing.assert_raises(
AssertionError,
numpy.testing.assert_array_equal,
test_value,
index[index_to_remove]
)
def test_data_dir_is_not_a_dir(self):
"""Unit test for __init__, data dir. as file case."""
with self.assertRaises(OSError):
Index(
TEST_VECTOR_SIZE,
os.path.join(TEST_INDEX_PATH, 'pupyl.index')
)
def test_index():
"""Unit test for method index."""
test_position = 0
with Index(TEST_VECTOR_SIZE, TEST_INDEX_PATH) as index:
test_value = index[test_position]
assert index.index(test_value) == 0
def test_group_by_position():
"""Unit test for method group_by."""
expected_result = [25, 89, 88, 44, 38, 64, 10, 101, 78, 67]
with Index(4, TEST_INDEX_SEARCH_PATH) as index:
test_result = [*index.group_by(position=0)][0]
assert expected_result == test_result
def test_data_dir_for_volatile_file(self):
"""Unit test for __init__, data dir for volatile file case."""
with self.assertRaises(DataDirDefinedForVolatileIndex):
Index(
TEST_VECTOR_SIZE,
data_dir=os.path.join(TEST_INDEX_PATH, 'pupyl.index'),
volatile=True
)
def test_group_by():
"""Unit test for method group_by."""
expected_result = {0: [25]}
with Index(4, TEST_INDEX_SEARCH_PATH) as index:
test_result = [*index.group_by(top=1)][0]
assert expected_result == test_result
def test_search():
"""Unit test for method search."""
query_array = [-0.48870765, -0.57780915, -0.94986234, -1.90035123]
expected_search_result = [25, 78]
with Index(len(query_array), TEST_INDEX_SEARCH_PATH) as index:
test_result = [*index.search(query_array, results=2)]
assert expected_search_result == test_result
def index(self, uri, **kwargs):
"""
Performs image indexing.
Parameters
----------
uri: str
Directory or file, or http(s) location.
**check_unique (optional): bool
If, during the index process, imported images
should have their unicity verified (to avoid duplicates).
"""
with Extractors(
characteristics=self._characteristic) as extractor, Index(
extractor.output_shape, data_dir=self._data_dir) as index:
self._index_configuration('w', feature_size=extractor.output_shape)
with concurrent.futures.ThreadPoolExecutor() as executor:
futures = {
executor.submit(self.image_database.insert, rank,
uri_from_file): rank
for rank, uri_from_file in enumerate(
extractor.scan_images(uri))
}
ranks = []
for future in extractor.progress(
concurrent.futures.as_completed(futures),
message='Importing images:'):
ranks.append(futures[future])
for rank in extractor.progress(sorted(ranks),
precise=True,
message='Indexing images:'):
features_tensor_name = self.image_database.\
mount_file_name(
rank,
'npy'
)
extractor.save_tensor(
extractor.extract,
self.image_database.mount_file_name(rank, 'jpg'),
features_tensor_name)
check_unique = kwargs.get('check_unique')
if check_unique is None:
check_unique = False
index.append(extractor.load_tensor(features_tensor_name),
check_unique=check_unique)
os.remove(features_tensor_name)
def test_pop():
"""Unit test for method pop."""
temp_file = FileIO.safe_temp_file(file_name='pupyl.index')
temp_dir = os.path.dirname(temp_file)
with Index(TEST_VECTOR_SIZE, data_dir=temp_dir) as index:
for _ in range(16):
index.append(numpy.random.normal(size=TEST_VECTOR_SIZE))
test_size_before = len(index)
test_value_before = index[-1]
with Index(TEST_VECTOR_SIZE, data_dir=temp_dir) as index:
test_value_after = index.pop()
assert len(index) == test_size_before - 1
numpy.testing.assert_array_equal(
test_value_before,
test_value_after
)
def test_append_new_file():
"""Unit test for method append, new file case."""
with Index(TEST_VECTOR_SIZE, volatile=True) as index:
test_size_before = len(index)
new_tensor = numpy.random.normal(size=TEST_VECTOR_SIZE)
index.append(new_tensor)
test_size_after = len(index)
assert test_size_after == test_size_before + 1
numpy.testing.assert_array_almost_equal(
index[-1], new_tensor, decimal=7
)
def test_export_group_by_position():
"""Unit test for method export_group_by method, position case."""
test_vector_size = 2560
test_files = set(f'{fname}.jpg' for fname in range(16))
test_files.add('group.jpg')
with Index(test_vector_size, data_dir=TEST_INDEX_EXPORT) as index:
for _ in range(2):
index.append(numpy.random.normal(size=test_vector_size))
with tempfile.TemporaryDirectory() as temp_dir:
index.export_by_group_by(temp_dir, top=1, position=0)
for _, _, files in os.walk(temp_dir):
for ffile in files:
assert ffile in test_files
def search(self, query, top=4):
"""
Executes the search for a created database
Parameters
----------
query: str
URI of a image to query
top (optional)(default: 4): int
How many results should be returned.
"""
with Extractors(characteristics=self._characteristic) as extractor:
with Index(extractor.output_shape,
data_dir=self._data_dir) as index:
for result in index.search(extractor.extract(query),
results=top):
yield result
def test_append_check_unique():
"""Unit test for method append, check unique case."""
with warnings.catch_warnings(record=True) as caught_warnings:
warnings.simplefilter('always')
with Extractors(
Characteristics.LIGHTWEIGHT_REGULAR_PRECISION
) as extractor, \
Index(extractor.output_shape, volatile=True) as index:
for image in extractor.scan(TEST_CHECK_UNIQUE):
index.append(
extractor.extract(image),
check_unique=True
)
assert len(caught_warnings) == 1
assert issubclass(caught_warnings[-1].category, UserWarning)
assert str(
caught_warnings[-1].message
) == 'Tensor being indexed already exists in the database ' + \
'and the check for duplicates are on. Refusing to store ' + \
'again this tensor.'
def test_empty_index(self):
"""Unit test for top parameter, negative case."""
with self.assertRaises(EmptyIndexError):
with Index(1, TEST_EMPTY_INDEX) as index:
_ = [*index.group_by()]
def test_raises_null_tensor_error(self):
"""Unit test for append, null tensor case."""
with self.assertRaises(NullTensorError):
with Index(TEST_VECTOR_SIZE, volatile=True) as index:
index.append(numpy.zeros(shape=TEST_VECTOR_SIZE))
def test_negative_top(self):
"""Unit test for top parameter, negative case."""
with self.assertRaises(TopNegativeOrZero):
with Index(4, TEST_INDEX_SEARCH_PATH) as index:
_ = [*index.group_by(top=-1)]
def test_no_data_dir_for_perm_file(self):
"""Unit test for __init__, no data dir for perm. file case."""
with self.assertRaises(NoDataDirForPermanentIndex):
Index(TEST_VECTOR_SIZE, data_dir=None)
from pupyl.indexer.exceptions import FileIsNotAnIndex, IndexNotBuildYet, \
NoDataDirForPermanentIndex, DataDirDefinedForVolatileIndex, \
NullTensorError, TopNegativeOrZero, EmptyIndexError
from pupyl.duplex.file_io import FileIO
from pupyl.embeddings.features import Extractors, Characteristics
TEST_INDEX_PATH = os.path.abspath('tests')
TEST_INDEX_SEARCH_PATH = os.path.join(TEST_INDEX_PATH, 'test_search')
TEST_INDEX_INVALID_FILE = os.path.join(TEST_INDEX_PATH, 'test_index')
TEST_INDEX_EXPORT = os.path.join(TEST_INDEX_PATH, 'test_index_export')
TEST_EMPTY_INDEX = os.path.join(TEST_INDEX_PATH, 'test_empty_index')
TEST_CHECK_UNIQUE = os.path.join(TEST_INDEX_PATH, 'test_check_unique')
TEST_VECTOR_SIZE = 128
INDEX = Index(TEST_VECTOR_SIZE, TEST_INDEX_PATH)
ALL_ITEMS = list(INDEX.items())
ALL_VALUES = list(INDEX.values())
class TestCases(TestCase):
"""Unit tests over special cases."""
def test___next_stop_iteration(self):
"""Unit test for method __next__, StopIteration case."""
with self.assertRaises(StopIteration):
index = Index(TEST_VECTOR_SIZE, TEST_INDEX_PATH)
while True:
next(index)
def test_open_invalid_file(self):
def test_context():
"""Unit test to test context opening and close."""
with Index(TEST_VECTOR_SIZE, TEST_INDEX_PATH) as index:
assert isinstance(index, Index)
def test_remove_new_file(self):
"""Unit test for method remove, new file case."""
with self.assertRaises(IndexNotBuildYet):
with Index(size=8, volatile=True) as index:
index.remove(0)
def test_open_invalid_file(self):
"""Unit test for method __init__, invalid file case."""
with self.assertRaises(FileIsNotAnIndex):
Index(TEST_VECTOR_SIZE, TEST_INDEX_INVALID_FILE)
def test_open_new_index():
"""Unit test for new index opening."""
with Index(TEST_VECTOR_SIZE, volatile=True) as index:
assert index.size == TEST_VECTOR_SIZE
def test___next_stop_iteration(self):
"""Unit test for method __next__, StopIteration case."""
with self.assertRaises(StopIteration):
index = Index(TEST_VECTOR_SIZE, TEST_INDEX_PATH)
while True:
next(index)