def test_ote_hpo(template):
work_dir, template_work_dir, _ = get_some_vars(template, root)
if os.path.exists(f"{template_work_dir}/hpo"):
shutil.rmtree(f"{template_work_dir}/hpo")
command_line = ['ote',
'train',
template.model_template_id,
'--train-ann-file',
f'{os.path.join(ote_dir, args["--train-ann-file"])}',
'--train-data-roots',
f'{os.path.join(ote_dir, args["--train-data-roots"])}',
'--val-ann-file',
f'{os.path.join(ote_dir, args["--val-ann-file"])}',
'--val-data-roots',
f'{os.path.join(ote_dir, args["--val-data-roots"])}',
'--save-model-to',
f'{template_work_dir}/hpo_trained_{template.model_template_id}',
'--enable-hpo',
'--hpo-time-ratio',
'1',
'params',
'--learning_parameters.num_iters',
'2',
'--learning_parameters.batch_size',
'2']
assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0
assert os.path.exists(f"{template_work_dir}/hpo/hpopt_status.json")
with open(f"{template_work_dir}/hpo/hpopt_status.json", "r") as f:
assert json.load(f).get('best_config_id', None) is not None
assert os.path.exists(f'{template_work_dir}/hpo_trained_{template.model_template_id}/weights.pth')
assert os.path.exists(f'{template_work_dir}/hpo_trained_{template.model_template_id}/label_schema.json')
def test_ote_deploy_openvino(template):
work_dir, template_work_dir, _ = get_some_vars(template, root)
deployment_dir = f'{template_work_dir}/deployed_{template.model_template_id}'
command_line = ['ote',
'deploy',
template.model_template_id,
'--load-weights',
f'{template_work_dir}/exported_{template.model_template_id}/openvino.xml',
f'--save-model-to',
deployment_dir]
assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0
assert run(['unzip', 'openvino.zip'],
cwd=deployment_dir).returncode == 0
assert run(['python3', '-m', 'venv', 'venv'],
cwd=os.path.join(deployment_dir, 'python')).returncode == 0
assert run(['python3', '-m', 'pip', 'install', 'wheel'],
cwd=os.path.join(deployment_dir, 'python'),
env=collect_env_vars(os.path.join(deployment_dir, 'python'))).returncode == 0
assert run(['python3', '-m', 'pip', 'install', 'demo_package-0.0-py3-none-any.whl'],
cwd=os.path.join(deployment_dir, 'python'),
env=collect_env_vars(os.path.join(deployment_dir, 'python'))).returncode == 0
patch_demo_py(os.path.join(deployment_dir, 'python', 'demo.py'),
os.path.join(deployment_dir, 'python', 'demo_patched.py'))
assert run(['python3', 'demo_patched.py', '-m', '../model/model.xml', '-i', f'{os.path.join(ote_dir, args["--test-data-roots"])}'],
cwd=os.path.join(deployment_dir, 'python'),
env=collect_env_vars(os.path.join(deployment_dir, 'python'))).returncode == 0
def test_ote_eval_openvino(template):
work_dir, template_work_dir, _ = get_some_vars(template, root)
command_line = [
'ote', 'eval', template.model_template_id, '--test-ann-file',
f'{os.path.join(ote_dir, args["--test-ann-files"])}',
'--test-data-roots',
f'{os.path.join(ote_dir, args["--test-data-roots"])}',
'--load-weights',
f'{template_work_dir}/exported_{template.model_template_id}/openvino.xml',
'--save-performance',
f'{template_work_dir}/exported_{template.model_template_id}/performance.json'
]
assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0
assert os.path.exists(
f'{template_work_dir}/exported_{template.model_template_id}/performance.json'
)
with open(
f'{template_work_dir}/trained_{template.model_template_id}/performance.json'
) as read_file:
trained_performance = json.load(read_file)
with open(
f'{template_work_dir}/exported_{template.model_template_id}/performance.json'
) as read_file:
exported_performance = json.load(read_file)
for k in trained_performance.keys():
assert abs(
trained_performance[k] - exported_performance[k]
) / trained_performance[
k] <= 0.00, f"{trained_performance[k]=}, {exported_performance[k]=}"
def test_ote_demo_openvino(template):
work_dir, template_work_dir, _ = get_some_vars(template, root)
command_line = [
'ote', 'demo', template.model_template_id, '--load-weights',
f'{template_work_dir}/exported_{template.model_template_id}/openvino.xml',
'--input', f'{os.path.join(ote_dir, args["--test-data-roots"])}',
'--delay', '-1'
]
assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0
def test_ote_export(template):
work_dir, template_work_dir, _ = get_some_vars(template, root)
command_line = ['ote',
'export',
template.model_template_id,
'--load-weights',
f'{template_work_dir}/trained_{template.model_template_id}/weights.pth',
f'--save-model-to',
f'{template_work_dir}/exported_{template.model_template_id}']
assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0
assert os.path.exists(f'{template_work_dir}/exported_{template.model_template_id}/openvino.xml')
assert os.path.exists(f'{template_work_dir}/exported_{template.model_template_id}/openvino.bin')
assert os.path.exists(f'{template_work_dir}/exported_{template.model_template_id}/label_schema.json')
def test_ote_eval(template):
work_dir, template_work_dir, _ = get_some_vars(template, root)
command_line = ['ote',
'eval',
template.model_template_id,
'--test-ann-file',
f'{os.path.join(ote_dir, args["--test-ann-files"])}',
'--test-data-roots',
f'{os.path.join(ote_dir, args["--test-data-roots"])}',
'--load-weights',
f'{template_work_dir}/trained_{template.model_template_id}/weights.pth',
'--save-performance',
f'{template_work_dir}/trained_{template.model_template_id}/performance.json']
assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0
assert os.path.exists(f'{template_work_dir}/trained_{template.model_template_id}/performance.json')
def test_ote_train(template):
work_dir, template_work_dir, algo_backend_dir = get_some_vars(
template, root)
create_venv(algo_backend_dir, work_dir, template_work_dir)
command_line = [
'ote', 'train', template.model_template_id, '--train-ann-file',
f'{os.path.join(ote_dir, args["--train-ann-file"])}',
'--train-data-roots',
f'{os.path.join(ote_dir, args["--train-data-roots"])}',
'--val-ann-file', f'{os.path.join(ote_dir, args["--val-ann-file"])}',
'--val-data-roots',
f'{os.path.join(ote_dir, args["--val-data-roots"])}',
'--save-model-to',
f'{template_work_dir}/trained_{template.model_template_id}'
]
assert run(command_line, env=collect_env_vars(work_dir)).returncode == 0
assert os.path.exists(
f'{template_work_dir}/trained_{template.model_template_id}/weights.pth'
)
assert os.path.exists(
f'{template_work_dir}/trained_{template.model_template_id}/label_schema.json'
)
