def test_input_csv_from_strings():
with open(os.path.join(cur_dir, 'docs.csv')) as fp:
lines = fp.readlines()
result = list(DocumentArray.from_csv(lines))
assert len(result) == 2
assert isinstance(result[0], Document)
assert result[0].tags['source'] == 'testsrc'
def test_client_csv(restful, mocker, func_name):
with Flow(restful=restful).add() as f, open(
os.path.join(cur_dir, 'docs.csv')
) as fp:
mock = mocker.Mock()
getattr(f, f'{func_name}')(DocumentArray.from_csv(fp), on_done=mock)
mock.assert_called_once()
def hello_world(args):
"""
Execute the chatbot example.
:param args: arguments passed from CLI
"""
Path(args.workdir).mkdir(parents=True, exist_ok=True)
with ImportExtensions(
required=True,
help_text=
'this demo requires Pytorch and Transformers to be installed, '
'if you haven\'t, please do `pip install jina[torch,transformers]`',
):
import transformers, torch
assert [torch,
transformers] #: prevent pycharm auto remove the above line
targets = {
'covid-csv': {
'url': args.index_data_url,
'filename': os.path.join(args.workdir, 'dataset.csv'),
}
}
# download the data
download_data(targets, args.download_proxy, task_name='download csv data')
# now comes the real work
# load index flow from a YAML file
f = (Flow().add(uses=MyTransformer,
parallel=args.parallel).add(uses=MyIndexer,
workspace=args.workdir))
# index it!
with f, open(targets['covid-csv']['filename']) as fp:
f.index(DocumentArray.from_csv(fp, field_resolver={'question':
'text'}))
# switch to REST gateway at runtime
f.use_rest_gateway(args.port_expose)
url_html_path = 'file://' + os.path.abspath(
os.path.join(os.path.dirname(os.path.realpath(__file__)),
'static/index.html'))
try:
webbrowser.open(url_html_path, new=2)
except:
pass # intentional pass, browser support isn't cross-platform
finally:
default_logger.success(
f'You should see a demo page opened in your browser, '
f'if not, you may open {url_html_path} manually')
if not args.unblock_query_flow:
f.block()
def hello_world(args):
"""
Execute the multimodal example.
:param args: arguments passed from CLI
"""
Path(args.workdir).mkdir(parents=True, exist_ok=True)
with ImportExtensions(
required=True,
help_text=
'this demo requires Pytorch and Transformers to be installed, '
'if you haven\'t, please do `pip install jina[torch,transformers]`',
):
import transformers, torch, torchvision
assert [
torch,
transformers,
torchvision,
] #: prevent pycharm auto remove the above line
# args.workdir = '0bae16ce-5bb2-43be-bcd4-6f1969e8068f'
targets = {
'people-img': {
'url': args.index_data_url,
'filename': os.path.join(args.workdir, 'dataset.zip'),
}
}
# download the data
if not os.path.exists(targets['people-img']['filename']):
download_data(targets,
args.download_proxy,
task_name='download zip data')
with zipfile.ZipFile(targets['people-img']['filename'], 'r') as fp:
fp.extractall(args.workdir)
# this envs are referred in index and query flow YAMLs
os.environ['HW_WORKDIR'] = args.workdir
os.environ['PY_MODULE'] = os.path.abspath(
os.path.join(cur_dir, 'my_executors.py'))
# now comes the real work
# load index flow from a YAML file
# index it!
f = Flow.load_config('flow-index.yml')
with f, open(f'{args.workdir}/people-img/meta.csv', newline='') as fp:
f.index(inputs=DocumentArray.from_csv(fp),
request_size=10,
show_progress=True)
# search it!
f = Flow.load_config('flow-search.yml')
# switch to HTTP gateway
f.protocol = 'http'
f.port_expose = args.port_expose
url_html_path = 'file://' + os.path.abspath(
os.path.join(cur_dir, 'static/index.html'))
with f:
try:
webbrowser.open(url_html_path, new=2)
except:
pass # intentional pass, browser support isn't cross-platform
finally:
default_logger.info(
f'You should see a demo page opened in your browser, '
f'if not, you may open {url_html_path} manually')
if not args.unblock_query_flow:
f.block()
def test_input_csv_from_file():
with open(os.path.join(cur_dir, 'docs.csv')) as fp:
result = list(DocumentArray.from_csv(fp))
assert len(result) == 2
assert isinstance(result[0], Document)
assert result[0].tags['source'] == 'testsrc'
