def index(num_docs: int): num_docs = min( num_docs, len(glob(os.path.join(os.getcwd(), IMAGE_SRC), recursive=True))) flow = Flow(workspace="workspace")\ .add(uses={"jtype": "ImageCrafter", "with": {"target_size": 96, "img_mean": [0.485, 0.456, 0.406], "img_std": [0.229, 0.224, 0.225]}}) \ .add(uses=BigTransferEncoder) \ .add(uses={"jtype": "EmbeddingIndexer", "with": {"index_file_name": "image.json"}, "metas": {"name": "vec_idx"}}, name="vec_idx") \ .add(uses={"jtype": "KeyValueIndexer", "metas": {"name": "kv_idx"}}, name="kv_idx", needs="gateway") \ .add(name="join_all", needs=["kv_idx", "vec_idx"], read_only="true") with flow: document_generator = from_files(IMAGE_SRC, size=num_docs) flow.post(on='/index', inputs=DocumentArray(document_generator), request_size=64, read_mode='rb')
def test_mime_type(protocol): f = Flow(protocol=protocol).add(uses=MimeExec) def validate_mime_type(req): for d in req.data.docs: assert d.mime_type == 'text/x-python' with f: f.index(from_files('*.py'), validate_mime_type)
def test_input_files(patterns, recursive, size, sampling_rate, read_mode): Client.check_input( from_files( patterns=patterns, recursive=recursive, size=size, sampling_rate=sampling_rate, read_mode=read_mode, ))
def test_image_convert_pipe(pytestconfig): for d in from_files(f'{pytestconfig.rootdir}/.github/**/*.png'): (d.convert_uri_to_image_blob().convert_uri_to_datauri(). set_image_blob_shape( (64, 64)).set_image_blob_normalization().set_image_blob_channel_axis( -1, 0)) assert d.blob.shape == (3, 64, 64) assert d.uri
def test_video_convert_pipe(pytestconfig, tmpdir): num_d = 0 for d in from_files(f'{pytestconfig.rootdir}/docs/**/*.mp4'): fname = str(tmpdir / f'tmp{num_d}.mp4') d.convert_uri_to_video_blob() d.dump_video_blob_to_file(fname) assert os.path.exists(fname) num_d += 1 assert num_d
def index(num_docs: int): # Runs indexing for all images num_docs = min( num_docs, len(glob(os.path.join(os.getcwd(), IMAGE_SRC), recursive=True))) with Flow.load_config('flows/index.yml') as flow: document_generator = from_files(IMAGE_SRC, size=num_docs) flow.post(on='/index', inputs=DocumentArray(document_generator), request_size=64, read_mode='rb')
def test_from_files_with_tilde(): shutil.copy( os.path.join(cur_dir, 'docs_groundtruth.jsonlines'), os.path.expanduser('~/'), ) shutil.copy( os.path.join(cur_dir, 'docs.csv'), os.path.expanduser('~/'), ) generator = from_files(patterns='~/*.*', to_dataturi=True, size=10) first = next(generator) assert first
def test_mime_type(protocol): class MyExec(Executor): @req def foo(self, docs: 'DocumentArray', **kwargs): for d in docs: d.convert_uri_to_buffer() f = Flow(protocol=protocol).add(uses=MyExec) def validate_mime_type(req): for d in req.data.docs: assert d.mime_type == 'text/x-python' with f: f.index(from_files('*.py'), validate_mime_type)
def test_sprite_image_generator(pytestconfig, tmpdir): da = DocumentArray( from_files( [ f'{pytestconfig.rootdir}/.github/**/*.png', f'{pytestconfig.rootdir}/.github/**/*.jpg', ] ) ) da.plot_image_sprites(tmpdir / 'sprint_da.png') assert os.path.exists(tmpdir / 'sprint_da.png') dam = DocumentArrayMemmap() dam.extend(da) dam.plot_image_sprites(tmpdir / 'sprint_dam.png') assert os.path.exists(tmpdir / 'sprint_dam.png')
def test_input_files_with_invalid_read_mode(client): with pytest.raises(BadClientInput): client.check_input(from_files(patterns='*.*', read_mode='invalid'))
def test_from_files_with_uri(): for d in from_files(patterns='*.*', to_dataturi=True, size=10): assert d.uri.startswith('data:')