def main(task, num_docs):
config()
data_path = './MNIST_Data/test/t10k-images-idx3-ubyte'
if task == 'index':
workspace = os.environ['WORKDIR']
if os.path.exists(workspace):
print(
f'\n +---------------------------------------------------------------------------------+ \
\n | ������ | \
\n | The directory {workspace} already exists. Please remove it before indexing again. | \
\n | ������ | \
\n +---------------------------------------------------------------------------------+'
)
f = Flow().load_config('mnist-index.yml')
with f:
f.index_ndarray(load_mnist(data_path), size=num_docs, batch_size=2)
elif task == 'search':
result_html = []
f = Flow().load_config('mnist-query.yml')
with f:
f.search_ndarray(load_mnist(data_path),
size=num_docs,
batch_size=1,
output_fn=print_result,
top_k=10)
write_html('result.html')
else:
raise NotImplementedError(
f'unknown task: {task}. A valid task is either `index` or `search`.'
)
def test_diff_field(cb_on, x_type):
def validate(x):
assert isinstance(x, x_type)
f = Flow().add()
with f:
f.index_ndarray(np.random.random([5, 4]),
output_fn=validate,
callback_on=cb_on)
def test_on_error():
def validate(x):
raise NotImplementedError
f = Flow().add()
with pytest.raises(BadClientCallback), f:
f.index_ndarray(np.random.random([5, 4]),
output_fn=validate,
continue_on_error=False)
with f:
f.index_ndarray(np.random.random([5, 4]),
output_fn=validate,
continue_on_error=True)
def test_unary_driver(self):
f = Flow().add(yaml_path='yaml/unarycrafter.yml')
def check_non_empty(req, field):
for d in req.index.docs:
self.assertEqual(len(d.chunks), 1)
self.assertEqual(d.chunks[0].WhichOneof('content'), field)
with f:
f.index_ndarray(np.random.random([10, 4, 2]),
output_fn=lambda x: check_non_empty(x, 'blob'))
with f:
f.index(np.random.random([10, 4, 2]),
output_fn=lambda x: check_non_empty(x, 'blob'))
with f:
f.index(['asda', 'dsadas asdasd'],
output_fn=lambda x: check_non_empty(x, 'text'))
def main(task, num_docs):
config()
data_path = './MNIST_Data/test/t10k-images-idx3-ubyte'
if task == 'index':
f = Flow().load_config('mnist-index.yml')
with f:
f.index_ndarray(load_mnist(data_path), size=num_docs, batch_size=2)
elif task == 'search':
result_html = []
f = Flow().load_config('mnist-query.yml')
with f:
f.search_ndarray(load_mnist(data_path),
size=num_docs,
batch_size=1,
output_fn=print_result,
top_k=10)
write_html('result.html')
else:
raise NotImplementedError(
f'unknown task: {task}. A valid task is either `index` or `search`.'
)
