def get_metrics(self) -> List[Dict[str, Any]]:
"""Get list of possible metrics."""
framework = self.config.get("framework", None)
if framework is None:
raise ClientErrorException("Framework not set.")
if framework == "pytorch":
check_module("ignite")
else:
check_module(framework)
help_dict = load_help_lpot_params("metrics")
key_in_framework_metrics = "onnxrt_qlinearops" if framework == "onnxrt" else framework
metrics_class = framework_metrics.get(key_in_framework_metrics)
raw_metric_list = list(
metrics_class().metrics.keys()) if metrics_class else []
raw_metric_list += ["custom"]
metrics_updated = _update_metric_parameters(raw_metric_list)
for metric, value in metrics_updated.copy().items():
if isinstance(value, dict):
for key in value.copy().keys():
help_msg_key = f"__help__{key}"
metrics_updated[metric][help_msg_key] = help_dict.get(
metric,
{},
).get(help_msg_key, "")
metrics_updated[f"__help__{metric}"] = help_dict.get(
f"__help__{metric}",
"",
)
return self._parse_help_in_dict(metrics_updated)
def get_strategies() -> List[Dict[str, Any]]:
"""Get list of supported strategies."""
help_dict = load_help_lpot_params("strategies")
strategies = []
for strategy in STRATEGIES.keys():
help_msg = help_dict.get(f"__help__{strategy}", "")
strategies.append({"name": strategy, "help": help_msg})
return strategies
def get_objectives() -> List[dict]:
"""Get list of supported objectives."""
help_dict = load_help_lpot_params("objectives")
objectives = []
for objective in OBJECTIVES.keys():
help_msg = help_dict.get(f"__help__{objective}", "")
objectives.append({"name": objective, "help": help_msg})
return objectives
def test_load_non_existing_help_lpot_params(self) -> None:
"""Test getting lpot strategies tooltips."""
with self.assertRaises(FileNotFoundError):
load_help_lpot_params("unknown_param")
def test_load_strategies_help_lpot_params(self) -> None:
"""Test getting lpot strategies tooltips."""
result = load_help_lpot_params("strategies")
self.assertIs(type(result), dict)
self.assertIsNot(result, {})
def test_load_metrics_help_lpot_params(self) -> None:
"""Test getting lpot metrics tooltips."""
result = load_help_lpot_params("metrics")
self.assertIs(type(result), dict)
self.assertIsNot(result, {})
