def test_shelf_in_flow(uses): m1 = used_memory() # shelve does not support embed > 1000?? # _dbm.error: cannot add item to database # HASH: Out of overflow pages. Increase page size docs = random_docs(10000, embed_dim=1000) f = Flow(callback_on='body').add(uses=os.path.join(cur_dir, uses)) with f: f.index(docs) m2 = used_memory() d = jina_pb2.DocumentProto() def validate(req): m4 = used_memory() print( f'before: {m1}, after index: {m2}, after loading: {m3} after searching {m4}' ) with f: m3 = used_memory() f.search([d], output_fn=validate) shutil.rmtree('test-workspace', ignore_errors=False, onerror=None)
def test_standard_query(tmpdir, test_standard): mem1 = used_memory(1) print(used_memory_readable()) with NumpyIndexer.load(os.path.join(tmpdir, 'a.bin')) as ni: ni.batch_size = 256 print(used_memory_readable()) print(ni.raw_ndarray.shape) print(used_memory_readable()) with TimeContext('query topk') as ti: result = ni.query(queries, top_k=10) mem2 = used_memory(1) print(used_memory_readable()) print(result[0].shape) with open(summary_file, 'a') as fp: json.dump( { 'name': 'naive', 'memory': mem2 - mem1, 'readable': get_readable_size(mem2 - mem1), 'time': ti.duration }, fp) fp.write('\n')
def format(self, record: 'LogRecord'): """ Format the log message as JSON object and add the current used memory. :param record: A LogRecord object. :return:: Return JSON formatted log if msg of LogRecord is dict type else return empty. """ from jina.logging.profile import used_memory cr = copy(record) if isinstance(cr.msg, dict): cr.msg.update({ k: getattr(cr, k) for k in ['created', 'module', 'process', 'thread'] }) cr.msg['memory'] = used_memory(unit=1) return json.dumps(cr.msg, sort_keys=True) else: return ''
def validate(req): m4 = used_memory() print( f'before: {m1}, after index: {m2}, after loading: {m3} after searching {m4}' )