def main(task, num_docs, request_size, data_set, model_name): config(model_name) workspace = os.environ['JINA_WORKSPACE'] logger = JinaLogger('cross-modal-search') if 'index' in task: if os.path.exists(workspace): logger.error( f'\n +------------------------------------------------------------------------------------+ \ \n | ������ | \ \n | The directory {workspace} already exists. Please remove it before indexing again. | \ \n | ������ | \ \n +------------------------------------------------------------------------------------+' ) sys.exit(1) logger.info(f'### task = {task}') if task == 'index': with Flow.load_config('flow-index.yml') as f: with TimeContext(f'QPS: indexing {num_docs}', logger=f.logger): f.index( input_fn=input_index_data(num_docs, request_size, data_set), request_size=request_size ) elif task == 'index_restful': index_restful(num_docs) elif task == 'query': with Flow.load_config('flow-query.yml') as f: f.use_rest_gateway() f.block() elif task == 'query_restful': if not os.path.exists(workspace): logger.warning(f'The directory {workspace} does not exist. Please index first via `python app.py -t index`') query_restful() elif task == 'dryrun': dryrun()
def index(data_set, num_docs, request_size): flow = Flow().load_config('flows/flow-index.yml') with flow: flow.post(on='/index', inputs=input_index_data(num_docs, request_size, data_set), request_size=request_size, show_progress=True)
def index(data_set, num_docs, request_size): f = Flow.load_config('flow-index.yml') with f: with TimeContext(f'QPS: indexing {num_docs}', logger=f.logger): f.index( inputs=input_index_data(num_docs, request_size, data_set), request_size=request_size )
def main(task, num_docs, request_size, data_set, model_name): config(model_name) if task == 'index': with Flow.load_config('flow-index.yml') as f: f.index(input_fn=input_index_data(num_docs, request_size, data_set), request_size=request_size) elif task == 'query': with Flow.load_config('flow-query.yml') as f: f.use_rest_gateway() f.block()
def main(index_num_docs, evaluate_num_docs, request_size, data_set, model_name, evaluation_mode): config(model_name) if index_num_docs > 0: with Flow.load_config('flow-index.yml') as f: f.index( input_fn=input_index_data(index_num_docs, request_size, data_set), request_size=request_size ) with Flow.load_config('flow-query.yml').add(name='evaluator', uses='yaml/evaluate.yml') as flow_eval: flow_eval.search( input_fn=evaluation_generator(evaluate_num_docs, request_size, data_set, mode=evaluation_mode), on_done=print_evaluation_score ) print(f'MeanReciprocalRank is: {sum_of_score / num_of_searches}')