def main(task, num_docs):
config(task)
workspace = os.environ['WORKDIR']
image_src = '/tmp/jina/celeb/lfw/**/*.jpg'
if task == 'index':
if os.path.exists(workspace):
print(
f'\n +---------------------------------------------------------------------------------+ \
\n | ������ | \
\n | The directory {workspace} already exists. Please remove it before indexing again. | \
\n | ������ | \
\n +---------------------------------------------------------------------------------+'
)
f = Flow().load_config('flow-index.yml')
with f:
f.index_files(image_src,
batch_size=8,
read_mode='rb',
size=num_docs)
elif task == 'query':
f = Flow().load_config('flow-query.yml')
f.use_rest_gateway()
with f:
f.block()
elif task == 'dryrun':
f = Flow.load_config('flow-query.yml')
with f:
pass
else:
raise NotImplementedError(
f'unknown task: {task}. A valid task is either `index` or `query` or `dryrun`.'
)
def main(task, num_docs, top_k):
workspace_path = '/tmp/jina/urbandict'
os.environ['TMP_WORKSPACE'] = get_random_ws(workspace_path)
print(f'{os.environ["TMP_WORKSPACE"]}')
data_fn = os.environ.get('WASHED_DATA_DIR', os.path.join(workspace_path, 'urbandict-word-defs.csv'))
if task == 'index':
f = Flow().load_config('flow-index.yml')
with f:
f.index_lines(filepath=data_fn, size=num_docs, batch_size=16)
elif task == 'query':
f = Flow().load_config('flow-query.yml')
with f:
while True:
text = input('word definition: ')
if not text:
break
ppr = lambda x: print_topk(x, text)
f.search_lines(lines=[text, ], output_fn=ppr, topk=top_k)
elif task == 'query_restful':
f = Flow().load_config('flow-query.yml')
f.use_rest_gateway()
with f:
f.block()
else:
raise NotImplementedError(
f'unknown task: {task}. A valid task is `index` or `query` or `query_restful`.')
def query_restful(return_flow=False):
f = Flow().load_config('flows/query.yml')
f.use_rest_gateway()
if return_flow:
return f
with f:
f.block()
def main(task, num_docs):
config()
data_path = os.path.join(os.environ['TMP_DATA_DIR'], 'jpg')
if task == 'index':
workspace = os.environ['WORKDIR']
if os.path.exists(workspace):
print(f'\n +---------------------------------------------------------------------------------+ \
\n | ������ | \
\n | The directory {workspace} already exists. Please remove it before indexing again. | \
\n | ������ | \
\n +---------------------------------------------------------------------------------+')
f = Flow().load_config('flow-index.yml')
with f:
f.index_files(f'{data_path}/*.jpg', size=num_docs, read_mode='rb', batch_size=2)
elif task == 'query':
f = Flow().load_config('flow-query.yml')
f.use_rest_gateway()
with f:
f.block()
elif task == 'dryrun':
f = Flow.load_config('flow-query.yml')
with f:
pass
else:
raise NotImplementedError(
f'unknown task: {task}. A valid task is either `index` or `query` or `dryrun`.')
def main(task, num_docs):
config()
data_path = os.path.join(os.environ['JINA_DATA_DIR'], 'jpg/*.jpg')
if task == 'index':
f = Flow().load_config('flow-index.yml')
with f:
f.index_files(data_path, batch_size=64, read_mode='rb', size=num_docs)
elif task == 'query':
f = Flow().load_config('flow-query.yml')
with f:
f.block()
else:
raise NotImplementedError(f'unknown task: {task}. A valid task is either `index` or `query`.')
def main(task):
if task == 'index':
f = Flow().load_config('flow-index.yml')
with f:
f.index(input_fn=data_fn)
elif task == 'query_restful':
f = Flow().load_config('flow-query.yml')
with f:
f.block()
else:
raise NotImplementedError(
f'unknown task: {task}. A valid task is `index` or `query` or `query_restful`.'
)
def main(task, num_docs):
os.environ['TMP_WORKSPACE'] = get_random_ws(os.environ['TMP_DATA_DIR'])
data_path = os.path.join(os.environ['TMP_DATA_DIR'], 'jpg')
if task == 'index':
f = Flow().load_config('flow-index.yml')
with f:
f.index_files(f'{data_path}/*.jpg', size=num_docs, read_mode='rb', batch_size=2)
elif task == 'query':
f = Flow().load_config('flow-query.yml')
f.use_rest_gateway()
with f:
f.block()
else:
raise NotImplementedError(f'unknown task: {task}. A valid task is either `index` or `query`.')
def main(task, num_docs, batch_size, top_k, data_set):
config()
if task == 'index':
f = Flow().load_config('flow-index.yml')
with f:
f.index(input_fn=input_index_data(num_docs, batch_size, data_set),
batch_size=batch_size)
elif task == 'query-restful':
# not working, missing a way to send modality via REST API
f = Flow().load_config('flow-query.yml')
f.use_rest_gateway()
with f:
f.block()
else:
raise NotImplementedError(
f'unknown task: {task}.'
f' A valid task is either `index`, `query-restful`, `query-i2t` and `query-t2i`.'
)
def main(task, num_docs):
config()
data_path = os.path.join(os.environ['TMP_DATA_DIR'], 'audio')
if task == 'index':
f = Flow().load_config('./flows/flow-index.yml')
with f:
f.index_files(f'{data_path}/*.wav', size=num_docs, batch_size=2)
elif task == 'query':
f = Flow().load_config('./flows/flow-query.yml')
f.use_rest_gateway()
with f:
f.block()
elif task == 'dryrun':
f = Flow.load_config('./flows/flow-query.yml')
with f:
pass
else:
raise NotImplementedError(
f'unknown task: {task}. A valid task is either `index` or `query` or `dryrun`.'
)
def main(task, num_docs, top_k):
workspace_path = '/tmp/jina/urbandict'
os.environ['WORKDIR'] = get_random_ws(workspace_path)
data_fn = os.environ.get(
'WASHED_DATA_DIR',
os.path.join(workspace_path, 'urbandict-word-defs.csv'))
if task == 'index':
workspace = os.environ['WORKDIR']
if os.path.exists(workspace):
print(
f'\n +---------------------------------------------------------------------------------+ \
\n | 🤖🤖🤖 | \
\n | The directory {workspace} already exists. Please remove it before indexing again. | \
\n | 🤖🤖🤖 | \
\n +---------------------------------------------------------------------------------+'
)
f = Flow().load_config('flow-index.yml')
with f:
f.index_lines(filepath=data_fn, size=num_docs, batch_size=16)
elif task == 'query':
f = Flow().load_config('flow-query.yml')
with f:
while True:
text = input('word definition: ')
if not text:
break
ppr = lambda x: print_topk(x, text)
f.search_lines(lines=[
text,
], output_fn=ppr, top_k=top_k)
elif task == 'query_restful':
f = Flow().load_config('flow-query.yml')
f.use_rest_gateway()
with f:
f.block()
else:
raise NotImplementedError(
f'unknown task: {task}. A valid task is `index` or `query` or `query_restful`.'
)
def main(task, num_docs):
config(task)
image_src = '/tmp/jina/celeb/lfw/**/*.jpg'
if task == 'index':
f = Flow().load_config('flow-index.yml')
with f:
f.index_files(image_src,
batch_size=8,
read_mode='rb',
size=num_docs)
elif task == 'query':
f = Flow().load_config('flow-query.yml')
f.use_rest_gateway()
with f:
f.block()
elif task == 'dryrun':
f = Flow.load_config('flow-query.yml')
with f:
pass
else:
raise NotImplementedError(
f'unknown task: {task}. A valid task is either `index` or `query` or `dryrun`.'
)
def search(workspace):
"""
Launch a neural search application to return the top-k search results
given an input prompt.
Requires install of package as stock[app]
"""
try:
from jina.flow import Flow
except ImportError:
raise ImportError(
"Jina is not installed. Did you install the package with .[app] extras?"
)
from stock import search, utils
yml = utils.resource_filename("flow-query.yml")
os.environ["SRC"] = os.path.dirname(yml)
os.environ["WORKSPACE"] = workspace
search.set_config()
flow = Flow().load_config(yml)
with flow:
with sp.Popen(["streamlit", "run", utils.resource_filename("app.py")]):
flow.block()
def query_restful():
f = Flow().load_config("flows/query.yml")
f.use_rest_gateway()
with f:
f.block()
def query():
f = Flow().load_config("flows/query.yml")
with f:
f.block() #printTopK()
def query_restful(num_docs):
config(num_docs, mode='search')
f = Flow().load_config('flow-query.yml')
f.use_rest_gateway()
with f:
f.block()
def query_restful(return_image):
f = Flow().load_config(f'flow-query-{return_image}.yml')
f.use_rest_gateway()
with f:
f.block()
def query_restful():
f = Flow().load_config('flow-query.yml')
f.use_rest_gateway()
with f:
f.block()
@click.option('--num_docs', '-n', default=50)
def main(task, num_docs):
config()
data_path = os.path.join(os.environ['WORKDIR'], 'audio')
if os.path.exists(data_path):
print(f'\n +---------------------------------------------------------------------------------+ \
\n | ������ | \
\n | The directory {data_path} already exists. Please remove it before indexing again. | \
\n | ������ | \
\n +---------------------------------------------------------------------------------+')
if task == 'index':
f = Flow().load_config('./flows/flow-index.yml')
with f:
f.index_files(f'{data_path}/*.wav', size=num_docs, batch_size=2)
elif task == 'query':
f = Flow().load_config('./flows/flow-query.yml')
f.use_rest_gateway()
with f:
f.block()
elif task == 'dryrun':
f = Flow.load_config('./flows/flow-query.yml')
with f:
pass
else:
raise NotImplementedError(
f'unknown task: {task}. A valid task is either `index` or `query` or `dryrun`.')
if __name__ == '__main__':
main()
def search():
f = Flow().load_config('flow-query.yml')
with f:
f.block()
