def test_optimizer_single_flow(tmpdir, config, sampler):
eval_flow_runner = SingleFlowRunner(
flow_yaml=os.path.join(cur_dir, 'flow.yml'),
documents=document_generator(10),
request_size=1,
execution_endpoint='search',
)
grid_sampler_search_space = {
'JINA_DUMMYCRAFTER_PARAM1': [0, 1],
'JINA_DUMMYCRAFTER_PARAM2': [0, 1, 2],
'JINA_DUMMYCRAFTER_PARAM3': [1],
}
opt = FlowOptimizer(
flow_runner=eval_flow_runner,
parameter_yaml=os.path.join(cur_dir, 'parameter.yml'),
evaluation_callback=EvaluationCallback(),
workspace_base_dir=str(tmpdir),
n_trials=5,
sampler=sampler,
)
if sampler == 'GridSampler':
result = opt.optimize_flow(search_space=grid_sampler_search_space)
else:
result = opt.optimize_flow()
validate_result(result, tmpdir)
def test_optimizer_multi_flow(tmpdir, config):
multi_flow_runner = MultiFlowRunner(
[
SingleFlowRunner(
flow_yaml=os.path.join(cur_dir, 'flow.yml'),
documents=document_generator(10),
request_size=1,
execution_endpoint='index',
),
SingleFlowRunner(
flow_yaml=os.path.join(cur_dir, 'flow.yml'),
documents=document_generator(10),
request_size=1,
execution_endpoint='search',
),
]
)
opt = FlowOptimizer(
flow_runner=multi_flow_runner,
parameter_yaml=os.path.join(cur_dir, 'parameter.yml'),
evaluation_callback=MeanEvaluationCallback(),
workspace_base_dir=str(tmpdir),
n_trials=5,
)
result = opt.optimize_flow()
validate_result(result, tmpdir)
def test_optimizer(tmpdir, config):
eval_flow_runner = SingleFlowRunner(
flow_yaml='tests/integration/optimizers/flow.yml',
documents=document_generator(10),
request_size=1,
execution_method='search',
)
opt = FlowOptimizer(
flow_runner=eval_flow_runner,
parameter_yaml='tests/integration/optimizers/parameter.yml',
evaluation_callback=MeanEvaluationCallback(),
workspace_base_dir=str(tmpdir),
n_trials=5,
)
result = opt.optimize_flow()
validate_result(result, tmpdir)
def test_optimizer_single_flow(tmpdir, config, sampler):
eval_flow_runner = SingleFlowRunner(
flow_yaml=os.path.join(cur_dir, 'flow.yml'),
documents=document_generator(10),
request_size=1,
execution_endpoint='search',
)
opt = FlowOptimizer(
flow_runner=eval_flow_runner,
parameter_yaml=os.path.join(cur_dir, 'parameter.yml'),
evaluation_callback=EvaluationCallback(),
workspace_base_dir=str(tmpdir),
n_trials=5,
sampler=sampler,
)
result = opt.optimize_flow()
validate_result(result, tmpdir)
def test_optimizer_single_flow_option2(tmpdir, config):
eval_flow_runner = SingleFlowRunner(
flow_yaml=os.path.join(cur_dir, 'flow_pod_choice.yml'),
documents=document_generator_option2(10),
request_size=1,
execution_endpoint='search',
)
opt = FlowOptimizer(
flow_runner=eval_flow_runner,
parameter_yaml=os.path.join(cur_dir, 'parameter_pod_choice.yml'),
evaluation_callback=EvaluationCallback(),
workspace_base_dir=str(tmpdir),
n_trials=20,
)
result = opt.optimize_flow()
assert (
result.best_parameters['JINA_DUMMYCRAFTER_CHOICE'] == 'pods/craft_option2.yml'
)
assert result.best_parameters['JINA_DUMMYCRAFTER_PARAM4'] == 0
assert result.best_parameters['JINA_DUMMYCRAFTER_PARAM5'] == 1
assert result.best_parameters['JINA_DUMMYCRAFTER_PARAM6'] == 1
def test_optimizer(tmpdir):
eval_flow_runner = SingleFlowRunner(
flow_yaml='tests/integration/optimizers/flow.yml',
documents=document_generator(10),
request_size=1,
task='search',
)
opt = FlowOptimizer(
flow_runner=eval_flow_runner,
parameter_yaml='tests/integration/optimizers/parameter.yml',
evaluation_callback=MeanEvaluationCallback(),
workspace_base_dir=str(tmpdir),
n_trials=5,
)
result = opt.optimize_flow()
result_path = str(tmpdir) + '/results/best_parameters.yml'
result.save_parameters(result_path)
parameters = result.best_parameters
assert parameters == BEST_PARAMETERS
assert yaml.load(open(result_path)) == BEST_PARAMETERS