def _migrate_workload(self, migration_version: int) -> None:
"""Migrate workload one version up."""
print(f"Migrate called with {migration_version} migration version.")
migrate = self.version_migrators.get(migration_version, None)
if migrate is None:
raise InternalException(
f"Could not parse workload from version {migration_version}")
migrate()
def test_from_exception_for_internal_exception(self) -> None:
"""Test from_exception for InternalException."""
message = "Domain code crashed!"
response = ResponseGenerator.from_exception(InternalException(message))
self.assertEqual(500, response.status_code)
self.assertEqual(message, response.data.decode("utf-8"))
def _migrate_workloads_list(self, migration_version: int) -> None:
"""Migrate workloads list one version up."""
migrate_workloads = self.version_migrators.get(migration_version, None)
if migrate_workloads is None:
raise InternalException(
f"Could not migrate workloads list to version {migration_version}",
)
migrate_workloads()
def get_parser(benchmark_mode: str, logs: List[str]) -> Parser:
"""Get benchmark parser for specified mode."""
parser_map = {
Benchmarks.PERF: PerformanceParser,
Benchmarks.ACC: AccuracyParser,
}
parser = parser_map.get(benchmark_mode, None)
if parser is None:
raise InternalException(
f"Could not find optimization class for {benchmark_mode}")
return parser(logs)
def get_optimization(workload: Workload,
template_path: Optional[str] = None) -> Optimization:
"""Get optimization for specified workload."""
optimization_map = {
Optimizations.TUNING: Tuning,
Optimizations.GRAPH: GraphOptimization,
}
optimization = optimization_map.get(workload.mode, None)
if optimization is None:
raise InternalException(
f"Could not find optimization class for {workload.mode}")
log.debug(f"Initializing {optimization.__name__} class.")
return optimization(workload, template_path)
def delete_workload(self, workload_id: str) -> None:
"""Delete workload by ID."""
if workload_id in self.workloads_data.get("workloads", {}):
try:
workload = self.workloads_data.get("workloads", {}).get(workload_id, {})
workload_path = workload.get("workload_path", None)
del self.workloads_data.get("workloads", {})[workload_id]
self.dump()
if workload_path:
shutil.rmtree(workload_path)
except Exception as e:
raise InternalException(
f"Error while removing workload {workload_id}:\n{str(e)}",
)
else:
raise NotFoundException(f"Can't find workload ID {workload_id}.")
def command(self, value: List[str]) -> None:
"""Set optimization command."""
if not isinstance(value, list):
raise InternalException("Command should be a list.")
self._command = value
def benchmark_model(
response_data: dict,
workload: Workload,
workdir: Workdir,
model: str,
model_path: str,
model_precision: str,
benchmark_mode: str,
benchmark_count: int,
benchmark_total: int,
) -> dict:
"""Benchmark model and prepare response data."""
request_id = response_data.get("id")
benchmark: Benchmark = Benchmark(
workload=workload,
model_path=model_path,
precision=model_precision,
mode=benchmark_mode,
)
log_name = f"{model}_{benchmark_mode}_benchmark"
executor = Executor(
workload.workload_path,
subject="benchmark",
data={"id": request_id},
send_response=False,
log_name=log_name,
additional_log_names=["output.txt"],
)
proc = executor.call(
benchmark.command,
)
logs = [os.path.join(workload.workload_path, f"{log_name}.txt")]
if not proc.is_ok:
raise ClientErrorException("Benchmark failed during execution.")
parser = BenchmarkParserFactory.get_parser(benchmark_mode, logs)
metrics = parser.process()
metric = {}
execution_details: Dict[str, Any] = {}
if benchmark_mode == Benchmarks.PERF:
result_field = f"perf_throughput_{model}"
elif benchmark_mode == Benchmarks.ACC:
result_field = f"acc_{model}"
else:
raise InternalException(f"Benchmark mode {benchmark_mode} is not supported.")
if isinstance(metrics, dict):
metric = {result_field: metrics.get(result_field, "")}
execution_details = response_data.get("execution_details", {})
model_benchmark_details = execution_details.get(f"{model}_benchmark", {})
model_benchmark_details.update(
{
benchmark_mode: benchmark.serialize(),
},
)
response_data.update({"progress": f"{benchmark_count}/{benchmark_total}"})
response_data.update(metric)
response_data["execution_details"].update(
{f"{model}_benchmark": model_benchmark_details},
)
workdir.update_metrics(
request_id=request_id,
metric_data=metric,
)
workdir.update_execution_details(
request_id=request_id,
execution_details=execution_details,
)
log.debug(f"Parsed data is {json.dumps(response_data)}")
mq.post_success("benchmark_progress", response_data)
return response_data