def index(num_docs: int):
num_docs = min(
num_docs,
len(glob(os.path.join(os.getcwd(), IMAGE_SRC), recursive=True)))
flow = Flow(workspace="workspace")\
.add(uses={"jtype": "ImageCrafter",
"with": {"target_size": 96,
"img_mean": [0.485, 0.456, 0.406],
"img_std": [0.229, 0.224, 0.225]}}) \
.add(uses=BigTransferEncoder) \
.add(uses={"jtype": "EmbeddingIndexer",
"with": {"index_file_name": "image.json"},
"metas": {"name": "vec_idx"}},
name="vec_idx") \
.add(uses={"jtype": "KeyValueIndexer",
"metas": {"name": "kv_idx"}},
name="kv_idx",
needs="gateway") \
.add(name="join_all",
needs=["kv_idx", "vec_idx"],
read_only="true")
with flow:
document_generator = from_files(IMAGE_SRC, size=num_docs)
flow.post(on='/index',
inputs=DocumentArray(document_generator),
request_size=64,
read_mode='rb')
def test_mime_type(protocol):
f = Flow(protocol=protocol).add(uses=MimeExec)
def validate_mime_type(req):
for d in req.data.docs:
assert d.mime_type == 'text/x-python'
with f:
f.index(from_files('*.py'), validate_mime_type)
def test_input_files(patterns, recursive, size, sampling_rate, read_mode):
Client.check_input(
from_files(
patterns=patterns,
recursive=recursive,
size=size,
sampling_rate=sampling_rate,
read_mode=read_mode,
))
def test_image_convert_pipe(pytestconfig):
for d in from_files(f'{pytestconfig.rootdir}/.github/**/*.png'):
(d.convert_uri_to_image_blob().convert_uri_to_datauri().
set_image_blob_shape(
(64,
64)).set_image_blob_normalization().set_image_blob_channel_axis(
-1, 0))
assert d.blob.shape == (3, 64, 64)
assert d.uri
def test_video_convert_pipe(pytestconfig, tmpdir):
num_d = 0
for d in from_files(f'{pytestconfig.rootdir}/docs/**/*.mp4'):
fname = str(tmpdir / f'tmp{num_d}.mp4')
d.convert_uri_to_video_blob()
d.dump_video_blob_to_file(fname)
assert os.path.exists(fname)
num_d += 1
assert num_d
def index(num_docs: int):
# Runs indexing for all images
num_docs = min(
num_docs,
len(glob(os.path.join(os.getcwd(), IMAGE_SRC), recursive=True)))
with Flow.load_config('flows/index.yml') as flow:
document_generator = from_files(IMAGE_SRC, size=num_docs)
flow.post(on='/index',
inputs=DocumentArray(document_generator),
request_size=64,
read_mode='rb')
def test_from_files_with_tilde():
shutil.copy(
os.path.join(cur_dir, 'docs_groundtruth.jsonlines'),
os.path.expanduser('~/'),
)
shutil.copy(
os.path.join(cur_dir, 'docs.csv'),
os.path.expanduser('~/'),
)
generator = from_files(patterns='~/*.*', to_dataturi=True, size=10)
first = next(generator)
assert first
def test_mime_type(protocol):
class MyExec(Executor):
@req
def foo(self, docs: 'DocumentArray', **kwargs):
for d in docs:
d.convert_uri_to_buffer()
f = Flow(protocol=protocol).add(uses=MyExec)
def validate_mime_type(req):
for d in req.data.docs:
assert d.mime_type == 'text/x-python'
with f:
f.index(from_files('*.py'), validate_mime_type)
def test_sprite_image_generator(pytestconfig, tmpdir):
da = DocumentArray(
from_files(
[
f'{pytestconfig.rootdir}/.github/**/*.png',
f'{pytestconfig.rootdir}/.github/**/*.jpg',
]
)
)
da.plot_image_sprites(tmpdir / 'sprint_da.png')
assert os.path.exists(tmpdir / 'sprint_da.png')
dam = DocumentArrayMemmap()
dam.extend(da)
dam.plot_image_sprites(tmpdir / 'sprint_dam.png')
assert os.path.exists(tmpdir / 'sprint_dam.png')
def test_input_files_with_invalid_read_mode(client):
with pytest.raises(BadClientInput):
client.check_input(from_files(patterns='*.*', read_mode='invalid'))
def test_from_files_with_uri():
for d in from_files(patterns='*.*', to_dataturi=True, size=10):
assert d.uri.startswith('data:')
