def main(task, num_docs, request_size, data_set, model_name):
config(model_name)
workspace = os.environ['JINA_WORKSPACE']
logger = JinaLogger('cross-modal-search')
if 'index' in task:
if os.path.exists(workspace):
logger.error(
f'\n +------------------------------------------------------------------------------------+ \
\n | ������ | \
\n | The directory {workspace} already exists. Please remove it before indexing again. | \
\n | ������ | \
\n +------------------------------------------------------------------------------------+'
)
sys.exit(1)
logger.info(f'### task = {task}')
if task == 'index':
with Flow.load_config('flow-index.yml') as f:
with TimeContext(f'QPS: indexing {num_docs}', logger=f.logger):
f.index(
input_fn=input_index_data(num_docs, request_size, data_set),
request_size=request_size
)
elif task == 'index_restful':
index_restful(num_docs)
elif task == 'query':
with Flow.load_config('flow-query.yml') as f:
f.use_rest_gateway()
f.block()
elif task == 'query_restful':
if not os.path.exists(workspace):
logger.warning(f'The directory {workspace} does not exist. Please index first via `python app.py -t index`')
query_restful()
elif task == 'dryrun':
dryrun()
def index(data_set, num_docs, request_size):
flow = Flow().load_config('flows/flow-index.yml')
with flow:
flow.post(on='/index',
inputs=input_index_data(num_docs, request_size, data_set),
request_size=request_size,
show_progress=True)
def index(data_set, num_docs, request_size):
f = Flow.load_config('flow-index.yml')
with f:
with TimeContext(f'QPS: indexing {num_docs}', logger=f.logger):
f.index(
inputs=input_index_data(num_docs, request_size, data_set),
request_size=request_size
)
def main(task, num_docs, request_size, data_set, model_name):
config(model_name)
if task == 'index':
with Flow.load_config('flow-index.yml') as f:
f.index(input_fn=input_index_data(num_docs, request_size,
data_set),
request_size=request_size)
elif task == 'query':
with Flow.load_config('flow-query.yml') as f:
f.use_rest_gateway()
f.block()
def main(index_num_docs, evaluate_num_docs, request_size, data_set, model_name, evaluation_mode):
config(model_name)
if index_num_docs > 0:
with Flow.load_config('flow-index.yml') as f:
f.index(
input_fn=input_index_data(index_num_docs, request_size, data_set),
request_size=request_size
)
with Flow.load_config('flow-query.yml').add(name='evaluator', uses='yaml/evaluate.yml') as flow_eval:
flow_eval.search(
input_fn=evaluation_generator(evaluate_num_docs, request_size, data_set, mode=evaluation_mode),
on_done=print_evaluation_score
)
print(f'MeanReciprocalRank is: {sum_of_score / num_of_searches}')