def validate_index_size(num_indexed_docs):
from jina.executors.compound import CompoundExecutor
path_compound = Path(
CompoundExecutor.get_component_workspace_from_compound_workspace(
os.environ['JINA_REST_DIR'], 'chunk_indexer', 0))
path = Path(os.environ['JINA_REST_DIR'])
bin_files = list(path_compound.glob('*.bin')) + list(path.glob('*.bin'))
assert len(bin_files) > 0
for index_file in bin_files:
index = BaseIndexer.load(str(index_file))
assert index.size == num_indexed_docs
def check_indexers_size(chunks, nr_docs, field, tmp_path, same_content, shards,
post_op):
cache_indexer_path = tmp_path / 'cache.bin'
cache_full_size = 0
with BaseIndexer.load(cache_indexer_path) as cache:
assert isinstance(cache, DocIDCache)
cache_full_size = cache.size
print(f'cache size {cache.size}')
for indexer_fname in [KV_IDX_FILENAME, VEC_IDX_FILENAME]:
indexers_full_size = 0
for i in range(shards):
from jina.executors.compound import CompoundExecutor
compound_name = 'inc_docindexer' if KV_IDX_FILENAME in indexer_fname else 'inc_vecindexer'
workspace_folder = CompoundExecutor.get_component_workspace_from_compound_workspace(
tmp_path, compound_name, i + 1 if shards > 1 else 0)
indexer_path = os.path.join(
BaseIndexer.get_shard_workspace(
workspace_folder=workspace_folder,
workspace_name=indexer_fname.rstrip('.bin'),
pea_id=i + 1 if shards > 1 else 0), f'{indexer_fname}')
# in the configuration of content-hash / same_content=True
# there aren't enough docs to satisfy batch size, only 1 shard will have it
if os.path.exists(indexer_path):
with BaseIndexer.load(indexer_path) as indexer:
if indexer_fname == KV_IDX_FILENAME:
assert isinstance(indexer, BinaryPbIndexer)
else:
assert isinstance(indexer, NumpyIndexer)
indexers_full_size += indexer.size
if post_op == 'delete':
assert indexers_full_size == 0
assert cache_full_size == 0
else:
if field == 'content_hash' and same_content:
if chunks > 0:
# one content from Doc, one from chunk
expected = 2
assert indexers_full_size == expected
assert cache_full_size == 2
else:
assert indexers_full_size == 1
assert cache_full_size == 1
else:
nr_expected = (nr_docs + chunks * nr_docs) * 2 if post_op == 'index2' \
else nr_docs + chunks * nr_docs
assert indexers_full_size == nr_expected
assert cache_full_size == nr_expected