def test_shelf_in_flow(uses):
m1 = used_memory()
# shelve does not support embed > 1000??
# _dbm.error: cannot add item to database
# HASH: Out of overflow pages. Increase page size
docs = random_docs(10000, embed_dim=1000)
f = Flow(callback_on='body').add(uses=os.path.join(cur_dir, uses))
with f:
f.index(docs)
m2 = used_memory()
d = jina_pb2.DocumentProto()
def validate(req):
m4 = used_memory()
print(
f'before: {m1}, after index: {m2}, after loading: {m3} after searching {m4}'
)
with f:
m3 = used_memory()
f.search([d], output_fn=validate)
shutil.rmtree('test-workspace', ignore_errors=False, onerror=None)
def test_standard_query(tmpdir, test_standard):
mem1 = used_memory(1)
print(used_memory_readable())
with NumpyIndexer.load(os.path.join(tmpdir, 'a.bin')) as ni:
ni.batch_size = 256
print(used_memory_readable())
print(ni.raw_ndarray.shape)
print(used_memory_readable())
with TimeContext('query topk') as ti:
result = ni.query(queries, top_k=10)
mem2 = used_memory(1)
print(used_memory_readable())
print(result[0].shape)
with open(summary_file, 'a') as fp:
json.dump(
{
'name': 'naive',
'memory': mem2 - mem1,
'readable': get_readable_size(mem2 - mem1),
'time': ti.duration
}, fp)
fp.write('\n')
def format(self, record: 'LogRecord'):
"""
Format the log message as JSON object and add the current used memory.
:param record: A LogRecord object.
:return:: Return JSON formatted log if msg of LogRecord is dict type else return empty.
"""
from jina.logging.profile import used_memory
cr = copy(record)
if isinstance(cr.msg, dict):
cr.msg.update({
k: getattr(cr, k)
for k in ['created', 'module', 'process', 'thread']
})
cr.msg['memory'] = used_memory(unit=1)
return json.dumps(cr.msg, sort_keys=True)
else:
return ''
def validate(req):
m4 = used_memory()
print(
f'before: {m1}, after index: {m2}, after loading: {m3} after searching {m4}'
)