def _migrate_to_v2(self) -> None:
"""Parse workload from v1 to v2."""
print("Migrating workload.json to v2...")
new_data = {
"input_precision": "fp32",
"output_precision": "int8",
"mode": "tuning",
"tune": True,
"version": 2,
}
parsed_workload = deepcopy(self.workload_data)
parsed_workload.update(new_data)
try:
parsed_workload["config"]["tuning"].update(
{"objective": "performance"})
except KeyError:
log.debug("Could not set tuning objective.")
try:
input_nodes = self.workload_data["config"]["model"][
"inputs"].split(",")
except KeyError:
input_nodes = []
parsed_workload.update({"input_nodes": input_nodes})
try:
output_nodes = self.workload_data["config"]["model"][
"outputs"].split(",")
except KeyError:
output_nodes = []
parsed_workload.update({"output_nodes": output_nodes})
self.workload_data = parsed_workload
def dump(self) -> None:
"""Dump workload to yaml."""
json_path = os.path.join(self.workload_path, "workload.json")
with open(json_path, "w") as f:
json.dump(self.serialize(), f, indent=4)
log.debug(f"Successfully saved workload to {json_path}")
def unpack_archive(self, archive_path: str, filename: str) -> str:
"""Unpack archive and return path to unpacked model."""
self.mq.post_success(
"unpack_start",
{
"id": self.request_id,
},
)
log.debug(f"Unpacking {archive_path}")
if zipfile.is_zipfile(archive_path):
z = zipfile.ZipFile(archive_path)
z.extractall(self.download_dir)
elif tarfile.is_tarfile(archive_path):
t = tarfile.open(archive_path, "r:gz")
t.extractall(self.download_dir)
else:
message = "Could unpack an archive. Supported archive types are zip and tar.gz."
self.mq.post_error(
"unpack_finish",
{"message": message, "code": 404, "id": self.request_id},
)
raise ClientErrorException(message)
os.remove(archive_path)
unpacked_path = os.path.join(self.download_dir, filename)
self.mq.post_success(
"unpack_finish",
{"id": self.request_id, "path": unpacked_path},
)
log.debug(f"Model file has been extracted to {unpacked_path}")
return unpacked_path
def require_migration(self) -> bool:
"""Check if workload require migration."""
if not os.path.isfile(self.workload_json):
log.debug("Workload does not exits.")
return False
if self.current_version >= max(self.version_migrators.keys()):
log.debug("Workload already up to date.")
return False
return True
def get_available_models(
workspace_path: Optional[str]) -> List[Dict[str, Any]]:
"""Get available models from Examples."""
model_list = []
full_list = load_model_config()
for framework in SUPPORTED_FRAMEWORKS:
try:
framework_version = get_module_version(framework)
except Exception:
log.debug(f"Framework {framework} not installed.")
continue
log.debug(f"{framework} version is {framework_version}")
framework_dict = full_list[framework]
for domain, domain_dict in framework_dict.items():
if not isinstance(domain_dict, dict):
continue
for model, model_dict in domain_dict.items():
if not isinstance(model_dict, dict):
continue
if check_version(
framework_version,
model_dict.get("framework_version", []),
):
model_list.append(
{
"framework":
framework,
"domain":
domain,
"model":
model,
"yaml":
get_model_zoo_config_path(
workspace_path,
framework,
domain,
model,
model_dict,
),
"model_path":
get_model_zoo_model_path(
workspace_path,
framework,
domain,
model,
model_dict,
),
}, )
validate_model_list(model_list)
return model_list
def get_boundary_nodes(data: Dict[str, Any]) -> None:
"""Get configuration."""
from lpot.ux.utils.utils import find_boundary_nodes
request_id = str(data.get("id", ""))
model_path = data.get("model_path", None)
if not (request_id and model_path):
message = "Missing model path or request id."
mq.post_error(
"boundary_nodes_finish",
{"message": message, "code": 404, "id": request_id},
)
return
try:
mq.post_success(
"boundary_nodes_start",
{"message": "started", "id": request_id},
)
model_repository = ModelRepository()
try:
model = model_repository.get_model(model_path)
except NotFoundException:
supported_frameworks = model_repository.get_frameworks()
raise ClientErrorException(
f"Framework for specified model is not yet supported. "
f"Supported frameworks are: {', '.join(supported_frameworks)}.",
)
framework = model.get_framework_name()
try:
check_module(framework)
except ClientErrorException:
raise ClientErrorException(
f"Detected {framework} model. "
f"Could not find installed {framework} module. "
f"Please install {framework}.",
)
framework_version = get_module_version(framework)
response_data = find_boundary_nodes(model_path)
response_data["id"] = request_id
response_data["framework"] = framework
response_data["framework_version"] = framework_version
except ClientErrorException as err:
mq.post_error(
"boundary_nodes_finish",
{"message": str(err), "code": 404, "id": request_id},
)
return
log.debug(f"Parsed data is {json.dumps(response_data)}")
mq.post_success("boundary_nodes_finish", response_data)
def get_optimization(workload: Workload,
template_path: Optional[str] = None) -> Optimization:
"""Get optimization for specified workload."""
optimization_map = {
Optimizations.TUNING: Tuning,
Optimizations.GRAPH: GraphOptimization,
}
optimization = optimization_map.get(workload.mode, None)
if optimization is None:
raise InternalException(
f"Could not find optimization class for {workload.mode}")
log.debug(f"Initializing {optimization.__name__} class.")
return optimization(workload, template_path)
def add_edge(self, source_id: str, target_id: str) -> bool:
"""Add an Edge to graph."""
try:
source = self.get_node(source_id)
target = self.get_node(target_id)
except NotFoundException as err:
log.debug(
f"Got an error: {str(err)} while attempted "
f"to add an Edge from {source_id} to {target_id}",
)
return False
self._edges.append(Edge(source, target))
return True
def process(self) -> Dict[str, Any]:
"""Process files."""
for log_file in self._logs:
log.debug(f"Read from {log_file}")
with open(log_file) as f:
for line in f:
for key in self.patterns:
prog = re.compile(self.patterns[key])
match = prog.search(line)
if match:
self.metric.insert_data(key, match.group(1))
parsed_data: Dict[str, Any] = self.metric.serialize() # type: ignore
return parsed_data
def __init__(
self,
output_dir: str = ".",
pid: Optional[str] = None,
request_id: Optional[str] = None,
filename: Optional[Union[str, List[str]]] = None,
additional_log_names: List[str] = [],
) -> None:
"""
Initialize class parameters.
:param output_dir: path to directory where process output saved
:param pid unique aibt process identifier
:return: the constructor returns no value
"""
pid = pid if pid else uuid.uuid4().hex
if filename:
name = filename
elif request_id:
name = request_id
else:
name = pid
log_names = [f"{name}.txt"]
info_file_name = f"{name}.proc"
log_names.extend(additional_log_names)
if not os.path.exists(output_dir):
try:
os.makedirs(output_dir)
except FileExistsError:
log.debug("Directory %s already exists.", output_dir)
# process output file
self.stage_output_path = os.path.join(output_dir, log_names[0])
self.log_paths = [
os.path.join(output_dir, filename) for filename in log_names
]
# process info file
self.proc_info_path = os.path.join(output_dir, info_file_name)
# process information
self.args: Optional[Iterable[Any]] = None
self.return_code: Optional[int] = None
self.time_start: Optional[datetime.datetime] = None
self.time_stop: Optional[datetime.datetime] = None
# default ignore exit code is only 0
self.ignore_exit_codes = [0]
def process(self) -> Dict[str, Any]:
"""Process accuracy logs."""
for log_file in self._logs:
log.debug(f"Read from {log_file}")
with open(log_file) as f:
for line in f:
for key in self.patterns:
prog = re.compile(self.patterns[key])
match = prog.search(line)
if match:
for precision in [
"input_model", "optimized_model"
]:
metric_name = f"acc_{precision}"
self.metric.insert_data(
metric_name, match.group(1))
parsed_data: Dict[str, Any] = self.metric.serialize() # type: ignore
return parsed_data
def process(self) -> Dict[str, Any]:
"""Process files."""
partial: Dict[str, List] = {}
for log_file in self._logs:
log.debug(f"Read from {log_file}")
with open(log_file) as f:
for line in f:
for key in self.patterns:
prog = re.compile(self.patterns[key])
match = prog.search(line)
if not match:
continue
metric_name = f"perf_{key}_input_model"
self.metric.insert_data(metric_name, match.group(1))
converted_value = getattr(self.metric, metric_name)
parse_result = {
key: converted_value,
}
partial = self.update_partial(partial, parse_result)
return self.summarize_partial(partial)
def execute_tuning(data: Dict[str, Any]) -> dict:
"""Get configuration."""
from lpot.ux.utils.workload.workload import Workload
if not str(data.get("id", "")):
message = "Missing request id."
mq.post_error(
"tuning_finish",
{
"message": message,
"code": 404
},
)
raise Exception(message)
request_id: str = data["id"]
workdir = Workdir(request_id=request_id)
workload_path: str = workdir.workload_path
try:
workload_data = load_json(os.path.join(workload_path,
"workload.json"), )
except Exception as err:
mq.post_error(
"tuning_finish",
{
"message": repr(err),
"code": 404,
"id": request_id
},
)
raise err
workload = Workload(workload_data)
tuning: Tuning = Tuning(workload, workdir.workload_path,
workdir.template_path)
send_data = {
"message": "started",
"id": request_id,
"size_fp32": get_size(tuning.model_path),
}
workdir.clean_logs()
workdir.update_data(
request_id=request_id,
model_path=tuning.model_path,
model_output_path=tuning.model_output_path,
status="wip",
)
executor = Executor(
workspace_path=workload_path,
subject="tuning",
data=send_data,
log_name="output",
)
proc = executor.call(tuning.command, )
tuning_time = executor.process_duration
if tuning_time:
tuning_time = round(tuning_time, 2)
log.debug(f"Elapsed time: {tuning_time}")
logs = [os.path.join(workload_path, "output.txt")]
parser = Parser(logs)
if proc.is_ok:
response_data = parser.process()
if isinstance(response_data, dict):
response_data["id"] = request_id
response_data["tuning_time"] = tuning_time
response_data["size_int8"] = get_size(tuning.model_output_path)
response_data["model_output_path"] = tuning.model_output_path
response_data["size_fp32"] = get_size(tuning.model_path)
response_data["is_custom_dataloader"] = bool(workdir.template_path)
workdir.update_data(
request_id=request_id,
model_path=tuning.model_path,
model_output_path=tuning.model_output_path,
metric=response_data,
status="success",
execution_details={"tuning": tuning.serialize()},
)
response_data["execution_details"] = {"tuning": tuning.serialize()}
log.debug(f"Parsed data is {json.dumps(response_data)}")
mq.post_success("tuning_finish", response_data)
return response_data
else:
log.debug("FAIL")
workdir.update_data(
request_id=request_id,
model_path=tuning.model_path,
status="error",
)
mq.post_failure("tuning_finish", {
"message": "failed",
"id": request_id
})
raise ClientErrorException("Tuning failed during execution.")
def call_one(
self,
args: List[Any],
logger: Optional[Any] = None,
executable: Optional[Any] = None,
shell: bool = False,
cwd: Optional[str] = None,
env: Optional[dict] = None,
universal_newlines: bool = False,
startupinfo: Optional[Any] = None,
creationflags: int = 0,
processes: Optional[Any] = None,
ignore_exit_codes: Union[list, Any] = None,
pid: Optional[str] = None,
) -> None:
"""
Execute single call for process.
:param args:
:param logger:
:param executable:
:param shell:
:param cwd:
:param env:
:param universal_newlines:
:param startupinfo:
:param creationflags:
:param processes:
:param ignore_exit_codes:
:param pid:
"""
proc = None
try:
log.debug("Exec %s ", " ".join(map(str, args)))
proc = Proc(
self.workdir,
pid=pid,
request_id=self.request_id,
filename=self.log_name,
additional_log_names=self.additional_log_names,
)
if self._send_response:
self._mq.post_success(
"_".join([self._subject, "start"]),
self.data,
)
proc.run(
args,
executable,
shell,
cwd,
env,
universal_newlines,
startupinfo,
creationflags,
ignore_exit_codes,
)
self.time_start = proc.time_start
self.time_stop = proc.time_stop
except Exception:
if self._send_response:
self._mq.post_error(
self._subject,
{
"message": Exception,
"id": self.request_id
},
)
log.exception("Unexpected error for command line %s", args)
try:
LOCK.acquire()
if processes is None:
processes = []
processes.append(proc)
finally:
LOCK.release()
log.debug("DONE")
def benchmark_model(
response_data: dict,
workload: Workload,
workdir: Workdir,
model: str,
model_path: str,
model_precision: str,
benchmark_mode: str,
benchmark_count: int,
benchmark_total: int,
) -> dict:
"""Benchmark model and prepare response data."""
request_id = response_data.get("id")
benchmark: Benchmark = Benchmark(
workload=workload,
model_path=model_path,
precision=model_precision,
mode=benchmark_mode,
)
log_name = f"{model}_{benchmark_mode}_benchmark"
executor = Executor(
workload.workload_path,
subject="benchmark",
data={"id": request_id},
send_response=False,
log_name=log_name,
additional_log_names=["output.txt"],
)
proc = executor.call(
benchmark.command,
)
logs = [os.path.join(workload.workload_path, f"{log_name}.txt")]
if not proc.is_ok:
raise ClientErrorException("Benchmark failed during execution.")
parser = BenchmarkParserFactory.get_parser(benchmark_mode, logs)
metrics = parser.process()
metric = {}
execution_details: Dict[str, Any] = {}
if benchmark_mode == Benchmarks.PERF:
result_field = f"perf_throughput_{model}"
elif benchmark_mode == Benchmarks.ACC:
result_field = f"acc_{model}"
else:
raise InternalException(f"Benchmark mode {benchmark_mode} is not supported.")
if isinstance(metrics, dict):
metric = {result_field: metrics.get(result_field, "")}
execution_details = response_data.get("execution_details", {})
model_benchmark_details = execution_details.get(f"{model}_benchmark", {})
model_benchmark_details.update(
{
benchmark_mode: benchmark.serialize(),
},
)
response_data.update({"progress": f"{benchmark_count}/{benchmark_total}"})
response_data.update(metric)
response_data["execution_details"].update(
{f"{model}_benchmark": model_benchmark_details},
)
workdir.update_metrics(
request_id=request_id,
metric_data=metric,
)
workdir.update_execution_details(
request_id=request_id,
execution_details=execution_details,
)
log.debug(f"Parsed data is {json.dumps(response_data)}")
mq.post_success("benchmark_progress", response_data)
return response_data
def download_file(
self,
url: str,
download_path: str,
headers: Optional[dict] = {},
) -> None:
"""Download specified file."""
try:
with requests.get(
url,
allow_redirects=True,
stream=True,
headers=headers,
) as r:
r.raise_for_status()
os.makedirs(os.path.dirname(download_path), exist_ok=True)
with open(download_path, "wb") as f:
log.debug(f"Download file from {url} to {download_path}")
total_length = r.headers.get("content-length")
self.mq.post_success(
"download_start",
{
"message": "started",
"id": self.request_id,
"url": url,
},
)
if total_length is None:
f.write(r.content)
return
downloaded = 0
last_progress = 0
total_size = int(total_length)
for data in r.iter_content(chunk_size=4096):
downloaded += len(data)
f.write(data)
if self.progress_steps:
progress = int(100 * downloaded / total_size)
if (last_progress != progress and progress %
int(100 / self.progress_steps) == 0):
self.mq.post_success(
"download_progress",
{
"id": self.request_id,
"progress":
f"{downloaded}/{total_size}",
},
)
log.debug(f"Download progress: {progress}%")
last_progress = progress
except requests.exceptions.HTTPError:
message = f"Error downloading file from {url} to {download_path}"
self.mq.post_error(
"download_finish",
{
"message": message,
"code": 404,
"id": self.request_id,
},
)
return
def run(
self,
args: List[Any],
executable: Optional[Any] = None,
shell: bool = False,
cwd: Optional[str] = None,
env: Optional[dict] = None,
universal_newlines: bool = False,
startupinfo: Optional[Any] = None,
creationflags: int = 0,
ignore_exit_codes: Union[list, Any] = None,
) -> subprocess.Popen:
"""
Execute call for process.
:param args:
:param executable:
:param shell:
:param cwd:
:param env:
:param universal_newlines:
:param startupinfo:
:param creationflags:
:param ignore_exit_codes:
:return: Popen
"""
self.time_start = datetime.datetime.utcnow()
self.time_stop = self.time_start
self.return_code = None
self.ignore_exit_codes = (ignore_exit_codes if ignore_exit_codes
and isinstance(ignore_exit_codes, list) else
[0])
try:
self.args = args
cmd: Union[str, Any] = " ".join(map(str, args)) if shell else map(
str, args)
proc = subprocess.Popen(
cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
shell=shell, # nosec
executable=executable,
cwd=cwd,
env=env,
universal_newlines=universal_newlines,
startupinfo=startupinfo,
creationflags=creationflags,
)
with ExitStack() as stack:
files = [
stack.enter_context(open(fname, "a", encoding="utf-8"))
for fname in self.log_paths
]
for line in proc.stdout: # type: ignore
decoded_line = line.decode("utf-8",
errors="ignore").strip()
log.debug(decoded_line)
for log_file in files:
log_file.write(decoded_line + "\n")
log_file.flush()
proc.wait()
self.time_stop = datetime.datetime.utcnow()
self.return_code = proc.returncode
msg = "Exit code: {}".format(self.return_code)
if self.return_code in self.ignore_exit_codes:
log.debug(msg)
else:
log.error(msg)
log.critical("...")
for line in self.tail:
log.critical(f"\t{line.strip()}")
relative_output_path = re.sub(
r"^.*stages/",
"stages/",
self.output_path,
)
log.critical(f"\tMore in file: {relative_output_path}")
log.critical("...")
return proc
finally:
self.__save_proc_info()
def execute_optimization(data: Dict[str, Any]) -> dict:
"""Get configuration."""
from lpot.ux.utils.workload.workload import Workload
if not str(data.get("id", "")):
message = "Missing request id."
mq.post_error(
"optimization_finish",
{"message": message, "code": 404},
)
raise Exception(message)
request_id: str = data["id"]
workdir = Workdir(request_id=request_id, overwrite=False)
workload_path: str = workdir.workload_path
try:
workload_data = _load_json_as_dict(
os.path.join(workload_path, "workload.json"),
)
except Exception as err:
mq.post_error(
"optimization_finish",
{"message": repr(err), "code": 404, "id": request_id},
)
raise err
workload = Workload(workload_data)
optimization: Optimization = OptimizationFactory.get_optimization(
workload,
workdir.template_path,
)
send_data = {
"message": "started",
"id": request_id,
"size_input_model": get_size(optimization.input_graph),
}
workdir.clean_logs()
workdir.update_data(
request_id=request_id,
model_path=optimization.input_graph,
input_precision=optimization.input_precision,
model_output_path=optimization.output_graph,
output_precision=optimization.output_precision,
status="wip",
)
executor = Executor(
workspace_path=workload_path,
subject="optimization",
data=send_data,
log_name="output",
)
proc = executor.call(
optimization.command,
)
optimization_time = executor.process_duration
if optimization_time:
optimization_time = round(optimization_time, 2)
log.debug(f"Elapsed time: {optimization_time}")
logs = [os.path.join(workload_path, "output.txt")]
parser = OptimizationParser(logs)
if proc.is_ok:
response_data = parser.process()
if isinstance(response_data, dict):
response_data["id"] = request_id
response_data["optimization_time"] = optimization_time
response_data["size_optimized_model"] = get_size(optimization.output_graph)
response_data["model_output_path"] = optimization.output_graph
response_data["size_input_model"] = get_size(optimization.input_graph)
response_data["is_custom_dataloader"] = bool(workdir.template_path)
workdir.update_data(
request_id=request_id,
model_path=optimization.input_graph,
model_output_path=optimization.output_graph,
metric=response_data,
status="success",
execution_details={"optimization": optimization.serialize()},
input_precision=optimization.input_precision,
output_precision=optimization.output_precision,
)
response_data["execution_details"] = {"optimization": optimization.serialize()}
log.debug(f"Parsed data is {json.dumps(response_data)}")
mq.post_success("optimization_finish", response_data)
return response_data
else:
log.debug("FAIL")
workdir.update_data(
request_id=request_id,
model_path=optimization.input_graph,
input_precision=optimization.input_precision,
output_precision=optimization.output_precision,
status="error",
)
mq.post_failure("optimization_finish", {"message": "failed", "id": request_id})
raise ClientErrorException("Optimization failed during execution.")
def load(self, path: str) -> None:
"""Load configuration from file."""
log.debug(f"Loading predefined config from {path}")
with open(path) as yaml_config:
config = yaml.safe_load(yaml_config)
self.initialize(config)
def execute_benchmark(data: Dict[str, Any]) -> None:
"""
Execute benchmark.
Expected data:
{
"id": "configuration_id",
"workspace_path": "/path/to/workspace",
"models": [
{
"precision": "fp32",
"path": "/localdisk/fp32.pb"
},
{
"precision": "int8",
"path": "/localdisk/int8.pb"
}
]
}
"""
from lpot.ux.utils.workload.workload import Workload
request_id = str(data.get("id", ""))
models = data.get("models", None)
if not (request_id and models):
message = "Missing request id or model list."
mq.post_error(
"benchmark_finish",
{"message": message, "code": 404, "id": request_id},
)
raise ClientErrorException(message)
workdir = Workdir(request_id=request_id, overwrite=False)
try:
workload_path = workdir.workload_path
workload_data = load_json(
os.path.join(workload_path, "workload.json"),
)
except Exception as err:
mq.post_error(
"benchmark_finish",
{"message": repr(err), "code": 404, "id": request_id},
)
raise ClientErrorException(repr(err))
workload = Workload(workload_data)
response_data: Dict[str, Any] = {"id": request_id, "execution_details": {}}
mq.post_success(
"benchmark_start",
{
"message": "started",
"id": request_id,
},
)
for idx, model_info in enumerate(models, start=1):
model_precision = model_info.get("precision", None)
model_path = model_info.get("path", None)
benchmark_mode = model_info.get("mode", "performance")
if not (model_precision and model_path):
message = "Missing model precision or model path."
mq.post_error(
"benchmark_finish",
{"message": message, "code": 404, "id": request_id},
)
raise ClientErrorException(message)
benchmark: Benchmark = Benchmark(
workload=workload,
model_path=model_path,
datatype=model_precision,
mode=benchmark_mode,
)
log_name = f"{model_precision}_{benchmark_mode}_benchmark"
executor = Executor(
workload_path,
subject="benchmark",
data={"id": request_id},
send_response=False,
log_name=log_name,
additional_log_names=["output.txt"],
)
proc = executor.call(
benchmark.command,
)
logs = [os.path.join(workload_path, f"{log_name}.txt")]
if proc.is_ok:
parser = Parser(logs)
metrics = parser.process()
metric = {}
execution_details: Dict[str, Any] = {}
throughput_field = f"perf_throughput_{model_precision}"
if isinstance(metrics, dict):
metric = {throughput_field: metrics.get(throughput_field, "")}
execution_details = {
f"{model_precision}_benchmark": benchmark.serialize(),
}
response_data.update({"progress": f"{idx}/{len(models)}"})
response_data.update(metric)
response_data["execution_details"].update(execution_details)
workdir.update_metrics(
request_id=request_id,
metric_data=metric,
)
workdir.update_execution_details(
request_id=request_id,
execution_details=execution_details,
)
log.debug(f"Parsed data is {json.dumps(response_data)}")
mq.post_success("benchmark_progress", response_data)
else:
log.error("Benchmark failed.")
mq.post_failure("benchmark_finish", {"message": "failed", "id": request_id})
raise ClientErrorException("Benchmark failed during execution.")
mq.post_success("benchmark_finish", response_data)
