def _migrate_to_v2(self) -> None: """Parse workload from v1 to v2.""" print("Migrating workload.json to v2...") new_data = { "input_precision": "fp32", "output_precision": "int8", "mode": "tuning", "tune": True, "version": 2, } parsed_workload = deepcopy(self.workload_data) parsed_workload.update(new_data) try: parsed_workload["config"]["tuning"].update( {"objective": "performance"}) except KeyError: log.debug("Could not set tuning objective.") try: input_nodes = self.workload_data["config"]["model"][ "inputs"].split(",") except KeyError: input_nodes = [] parsed_workload.update({"input_nodes": input_nodes}) try: output_nodes = self.workload_data["config"]["model"][ "outputs"].split(",") except KeyError: output_nodes = [] parsed_workload.update({"output_nodes": output_nodes}) self.workload_data = parsed_workload
def dump(self) -> None: """Dump workload to yaml.""" json_path = os.path.join(self.workload_path, "workload.json") with open(json_path, "w") as f: json.dump(self.serialize(), f, indent=4) log.debug(f"Successfully saved workload to {json_path}")
def unpack_archive(self, archive_path: str, filename: str) -> str: """Unpack archive and return path to unpacked model.""" self.mq.post_success( "unpack_start", { "id": self.request_id, }, ) log.debug(f"Unpacking {archive_path}") if zipfile.is_zipfile(archive_path): z = zipfile.ZipFile(archive_path) z.extractall(self.download_dir) elif tarfile.is_tarfile(archive_path): t = tarfile.open(archive_path, "r:gz") t.extractall(self.download_dir) else: message = "Could unpack an archive. Supported archive types are zip and tar.gz." self.mq.post_error( "unpack_finish", {"message": message, "code": 404, "id": self.request_id}, ) raise ClientErrorException(message) os.remove(archive_path) unpacked_path = os.path.join(self.download_dir, filename) self.mq.post_success( "unpack_finish", {"id": self.request_id, "path": unpacked_path}, ) log.debug(f"Model file has been extracted to {unpacked_path}") return unpacked_path
def require_migration(self) -> bool: """Check if workload require migration.""" if not os.path.isfile(self.workload_json): log.debug("Workload does not exits.") return False if self.current_version >= max(self.version_migrators.keys()): log.debug("Workload already up to date.") return False return True
def get_available_models( workspace_path: Optional[str]) -> List[Dict[str, Any]]: """Get available models from Examples.""" model_list = [] full_list = load_model_config() for framework in SUPPORTED_FRAMEWORKS: try: framework_version = get_module_version(framework) except Exception: log.debug(f"Framework {framework} not installed.") continue log.debug(f"{framework} version is {framework_version}") framework_dict = full_list[framework] for domain, domain_dict in framework_dict.items(): if not isinstance(domain_dict, dict): continue for model, model_dict in domain_dict.items(): if not isinstance(model_dict, dict): continue if check_version( framework_version, model_dict.get("framework_version", []), ): model_list.append( { "framework": framework, "domain": domain, "model": model, "yaml": get_model_zoo_config_path( workspace_path, framework, domain, model, model_dict, ), "model_path": get_model_zoo_model_path( workspace_path, framework, domain, model, model_dict, ), }, ) validate_model_list(model_list) return model_list
def get_boundary_nodes(data: Dict[str, Any]) -> None: """Get configuration.""" from lpot.ux.utils.utils import find_boundary_nodes request_id = str(data.get("id", "")) model_path = data.get("model_path", None) if not (request_id and model_path): message = "Missing model path or request id." mq.post_error( "boundary_nodes_finish", {"message": message, "code": 404, "id": request_id}, ) return try: mq.post_success( "boundary_nodes_start", {"message": "started", "id": request_id}, ) model_repository = ModelRepository() try: model = model_repository.get_model(model_path) except NotFoundException: supported_frameworks = model_repository.get_frameworks() raise ClientErrorException( f"Framework for specified model is not yet supported. " f"Supported frameworks are: {', '.join(supported_frameworks)}.", ) framework = model.get_framework_name() try: check_module(framework) except ClientErrorException: raise ClientErrorException( f"Detected {framework} model. " f"Could not find installed {framework} module. " f"Please install {framework}.", ) framework_version = get_module_version(framework) response_data = find_boundary_nodes(model_path) response_data["id"] = request_id response_data["framework"] = framework response_data["framework_version"] = framework_version except ClientErrorException as err: mq.post_error( "boundary_nodes_finish", {"message": str(err), "code": 404, "id": request_id}, ) return log.debug(f"Parsed data is {json.dumps(response_data)}") mq.post_success("boundary_nodes_finish", response_data)
def get_optimization(workload: Workload, template_path: Optional[str] = None) -> Optimization: """Get optimization for specified workload.""" optimization_map = { Optimizations.TUNING: Tuning, Optimizations.GRAPH: GraphOptimization, } optimization = optimization_map.get(workload.mode, None) if optimization is None: raise InternalException( f"Could not find optimization class for {workload.mode}") log.debug(f"Initializing {optimization.__name__} class.") return optimization(workload, template_path)
def add_edge(self, source_id: str, target_id: str) -> bool: """Add an Edge to graph.""" try: source = self.get_node(source_id) target = self.get_node(target_id) except NotFoundException as err: log.debug( f"Got an error: {str(err)} while attempted " f"to add an Edge from {source_id} to {target_id}", ) return False self._edges.append(Edge(source, target)) return True
def process(self) -> Dict[str, Any]: """Process files.""" for log_file in self._logs: log.debug(f"Read from {log_file}") with open(log_file) as f: for line in f: for key in self.patterns: prog = re.compile(self.patterns[key]) match = prog.search(line) if match: self.metric.insert_data(key, match.group(1)) parsed_data: Dict[str, Any] = self.metric.serialize() # type: ignore return parsed_data
def __init__( self, output_dir: str = ".", pid: Optional[str] = None, request_id: Optional[str] = None, filename: Optional[Union[str, List[str]]] = None, additional_log_names: List[str] = [], ) -> None: """ Initialize class parameters. :param output_dir: path to directory where process output saved :param pid unique aibt process identifier :return: the constructor returns no value """ pid = pid if pid else uuid.uuid4().hex if filename: name = filename elif request_id: name = request_id else: name = pid log_names = [f"{name}.txt"] info_file_name = f"{name}.proc" log_names.extend(additional_log_names) if not os.path.exists(output_dir): try: os.makedirs(output_dir) except FileExistsError: log.debug("Directory %s already exists.", output_dir) # process output file self.stage_output_path = os.path.join(output_dir, log_names[0]) self.log_paths = [ os.path.join(output_dir, filename) for filename in log_names ] # process info file self.proc_info_path = os.path.join(output_dir, info_file_name) # process information self.args: Optional[Iterable[Any]] = None self.return_code: Optional[int] = None self.time_start: Optional[datetime.datetime] = None self.time_stop: Optional[datetime.datetime] = None # default ignore exit code is only 0 self.ignore_exit_codes = [0]
def process(self) -> Dict[str, Any]: """Process accuracy logs.""" for log_file in self._logs: log.debug(f"Read from {log_file}") with open(log_file) as f: for line in f: for key in self.patterns: prog = re.compile(self.patterns[key]) match = prog.search(line) if match: for precision in [ "input_model", "optimized_model" ]: metric_name = f"acc_{precision}" self.metric.insert_data( metric_name, match.group(1)) parsed_data: Dict[str, Any] = self.metric.serialize() # type: ignore return parsed_data
def process(self) -> Dict[str, Any]: """Process files.""" partial: Dict[str, List] = {} for log_file in self._logs: log.debug(f"Read from {log_file}") with open(log_file) as f: for line in f: for key in self.patterns: prog = re.compile(self.patterns[key]) match = prog.search(line) if not match: continue metric_name = f"perf_{key}_input_model" self.metric.insert_data(metric_name, match.group(1)) converted_value = getattr(self.metric, metric_name) parse_result = { key: converted_value, } partial = self.update_partial(partial, parse_result) return self.summarize_partial(partial)
def execute_tuning(data: Dict[str, Any]) -> dict: """Get configuration.""" from lpot.ux.utils.workload.workload import Workload if not str(data.get("id", "")): message = "Missing request id." mq.post_error( "tuning_finish", { "message": message, "code": 404 }, ) raise Exception(message) request_id: str = data["id"] workdir = Workdir(request_id=request_id) workload_path: str = workdir.workload_path try: workload_data = load_json(os.path.join(workload_path, "workload.json"), ) except Exception as err: mq.post_error( "tuning_finish", { "message": repr(err), "code": 404, "id": request_id }, ) raise err workload = Workload(workload_data) tuning: Tuning = Tuning(workload, workdir.workload_path, workdir.template_path) send_data = { "message": "started", "id": request_id, "size_fp32": get_size(tuning.model_path), } workdir.clean_logs() workdir.update_data( request_id=request_id, model_path=tuning.model_path, model_output_path=tuning.model_output_path, status="wip", ) executor = Executor( workspace_path=workload_path, subject="tuning", data=send_data, log_name="output", ) proc = executor.call(tuning.command, ) tuning_time = executor.process_duration if tuning_time: tuning_time = round(tuning_time, 2) log.debug(f"Elapsed time: {tuning_time}") logs = [os.path.join(workload_path, "output.txt")] parser = Parser(logs) if proc.is_ok: response_data = parser.process() if isinstance(response_data, dict): response_data["id"] = request_id response_data["tuning_time"] = tuning_time response_data["size_int8"] = get_size(tuning.model_output_path) response_data["model_output_path"] = tuning.model_output_path response_data["size_fp32"] = get_size(tuning.model_path) response_data["is_custom_dataloader"] = bool(workdir.template_path) workdir.update_data( request_id=request_id, model_path=tuning.model_path, model_output_path=tuning.model_output_path, metric=response_data, status="success", execution_details={"tuning": tuning.serialize()}, ) response_data["execution_details"] = {"tuning": tuning.serialize()} log.debug(f"Parsed data is {json.dumps(response_data)}") mq.post_success("tuning_finish", response_data) return response_data else: log.debug("FAIL") workdir.update_data( request_id=request_id, model_path=tuning.model_path, status="error", ) mq.post_failure("tuning_finish", { "message": "failed", "id": request_id }) raise ClientErrorException("Tuning failed during execution.")
def call_one( self, args: List[Any], logger: Optional[Any] = None, executable: Optional[Any] = None, shell: bool = False, cwd: Optional[str] = None, env: Optional[dict] = None, universal_newlines: bool = False, startupinfo: Optional[Any] = None, creationflags: int = 0, processes: Optional[Any] = None, ignore_exit_codes: Union[list, Any] = None, pid: Optional[str] = None, ) -> None: """ Execute single call for process. :param args: :param logger: :param executable: :param shell: :param cwd: :param env: :param universal_newlines: :param startupinfo: :param creationflags: :param processes: :param ignore_exit_codes: :param pid: """ proc = None try: log.debug("Exec %s ", " ".join(map(str, args))) proc = Proc( self.workdir, pid=pid, request_id=self.request_id, filename=self.log_name, additional_log_names=self.additional_log_names, ) if self._send_response: self._mq.post_success( "_".join([self._subject, "start"]), self.data, ) proc.run( args, executable, shell, cwd, env, universal_newlines, startupinfo, creationflags, ignore_exit_codes, ) self.time_start = proc.time_start self.time_stop = proc.time_stop except Exception: if self._send_response: self._mq.post_error( self._subject, { "message": Exception, "id": self.request_id }, ) log.exception("Unexpected error for command line %s", args) try: LOCK.acquire() if processes is None: processes = [] processes.append(proc) finally: LOCK.release() log.debug("DONE")
def benchmark_model( response_data: dict, workload: Workload, workdir: Workdir, model: str, model_path: str, model_precision: str, benchmark_mode: str, benchmark_count: int, benchmark_total: int, ) -> dict: """Benchmark model and prepare response data.""" request_id = response_data.get("id") benchmark: Benchmark = Benchmark( workload=workload, model_path=model_path, precision=model_precision, mode=benchmark_mode, ) log_name = f"{model}_{benchmark_mode}_benchmark" executor = Executor( workload.workload_path, subject="benchmark", data={"id": request_id}, send_response=False, log_name=log_name, additional_log_names=["output.txt"], ) proc = executor.call( benchmark.command, ) logs = [os.path.join(workload.workload_path, f"{log_name}.txt")] if not proc.is_ok: raise ClientErrorException("Benchmark failed during execution.") parser = BenchmarkParserFactory.get_parser(benchmark_mode, logs) metrics = parser.process() metric = {} execution_details: Dict[str, Any] = {} if benchmark_mode == Benchmarks.PERF: result_field = f"perf_throughput_{model}" elif benchmark_mode == Benchmarks.ACC: result_field = f"acc_{model}" else: raise InternalException(f"Benchmark mode {benchmark_mode} is not supported.") if isinstance(metrics, dict): metric = {result_field: metrics.get(result_field, "")} execution_details = response_data.get("execution_details", {}) model_benchmark_details = execution_details.get(f"{model}_benchmark", {}) model_benchmark_details.update( { benchmark_mode: benchmark.serialize(), }, ) response_data.update({"progress": f"{benchmark_count}/{benchmark_total}"}) response_data.update(metric) response_data["execution_details"].update( {f"{model}_benchmark": model_benchmark_details}, ) workdir.update_metrics( request_id=request_id, metric_data=metric, ) workdir.update_execution_details( request_id=request_id, execution_details=execution_details, ) log.debug(f"Parsed data is {json.dumps(response_data)}") mq.post_success("benchmark_progress", response_data) return response_data
def download_file( self, url: str, download_path: str, headers: Optional[dict] = {}, ) -> None: """Download specified file.""" try: with requests.get( url, allow_redirects=True, stream=True, headers=headers, ) as r: r.raise_for_status() os.makedirs(os.path.dirname(download_path), exist_ok=True) with open(download_path, "wb") as f: log.debug(f"Download file from {url} to {download_path}") total_length = r.headers.get("content-length") self.mq.post_success( "download_start", { "message": "started", "id": self.request_id, "url": url, }, ) if total_length is None: f.write(r.content) return downloaded = 0 last_progress = 0 total_size = int(total_length) for data in r.iter_content(chunk_size=4096): downloaded += len(data) f.write(data) if self.progress_steps: progress = int(100 * downloaded / total_size) if (last_progress != progress and progress % int(100 / self.progress_steps) == 0): self.mq.post_success( "download_progress", { "id": self.request_id, "progress": f"{downloaded}/{total_size}", }, ) log.debug(f"Download progress: {progress}%") last_progress = progress except requests.exceptions.HTTPError: message = f"Error downloading file from {url} to {download_path}" self.mq.post_error( "download_finish", { "message": message, "code": 404, "id": self.request_id, }, ) return
def run( self, args: List[Any], executable: Optional[Any] = None, shell: bool = False, cwd: Optional[str] = None, env: Optional[dict] = None, universal_newlines: bool = False, startupinfo: Optional[Any] = None, creationflags: int = 0, ignore_exit_codes: Union[list, Any] = None, ) -> subprocess.Popen: """ Execute call for process. :param args: :param executable: :param shell: :param cwd: :param env: :param universal_newlines: :param startupinfo: :param creationflags: :param ignore_exit_codes: :return: Popen """ self.time_start = datetime.datetime.utcnow() self.time_stop = self.time_start self.return_code = None self.ignore_exit_codes = (ignore_exit_codes if ignore_exit_codes and isinstance(ignore_exit_codes, list) else [0]) try: self.args = args cmd: Union[str, Any] = " ".join(map(str, args)) if shell else map( str, args) proc = subprocess.Popen( cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=shell, # nosec executable=executable, cwd=cwd, env=env, universal_newlines=universal_newlines, startupinfo=startupinfo, creationflags=creationflags, ) with ExitStack() as stack: files = [ stack.enter_context(open(fname, "a", encoding="utf-8")) for fname in self.log_paths ] for line in proc.stdout: # type: ignore decoded_line = line.decode("utf-8", errors="ignore").strip() log.debug(decoded_line) for log_file in files: log_file.write(decoded_line + "\n") log_file.flush() proc.wait() self.time_stop = datetime.datetime.utcnow() self.return_code = proc.returncode msg = "Exit code: {}".format(self.return_code) if self.return_code in self.ignore_exit_codes: log.debug(msg) else: log.error(msg) log.critical("...") for line in self.tail: log.critical(f"\t{line.strip()}") relative_output_path = re.sub( r"^.*stages/", "stages/", self.output_path, ) log.critical(f"\tMore in file: {relative_output_path}") log.critical("...") return proc finally: self.__save_proc_info()
def execute_optimization(data: Dict[str, Any]) -> dict: """Get configuration.""" from lpot.ux.utils.workload.workload import Workload if not str(data.get("id", "")): message = "Missing request id." mq.post_error( "optimization_finish", {"message": message, "code": 404}, ) raise Exception(message) request_id: str = data["id"] workdir = Workdir(request_id=request_id, overwrite=False) workload_path: str = workdir.workload_path try: workload_data = _load_json_as_dict( os.path.join(workload_path, "workload.json"), ) except Exception as err: mq.post_error( "optimization_finish", {"message": repr(err), "code": 404, "id": request_id}, ) raise err workload = Workload(workload_data) optimization: Optimization = OptimizationFactory.get_optimization( workload, workdir.template_path, ) send_data = { "message": "started", "id": request_id, "size_input_model": get_size(optimization.input_graph), } workdir.clean_logs() workdir.update_data( request_id=request_id, model_path=optimization.input_graph, input_precision=optimization.input_precision, model_output_path=optimization.output_graph, output_precision=optimization.output_precision, status="wip", ) executor = Executor( workspace_path=workload_path, subject="optimization", data=send_data, log_name="output", ) proc = executor.call( optimization.command, ) optimization_time = executor.process_duration if optimization_time: optimization_time = round(optimization_time, 2) log.debug(f"Elapsed time: {optimization_time}") logs = [os.path.join(workload_path, "output.txt")] parser = OptimizationParser(logs) if proc.is_ok: response_data = parser.process() if isinstance(response_data, dict): response_data["id"] = request_id response_data["optimization_time"] = optimization_time response_data["size_optimized_model"] = get_size(optimization.output_graph) response_data["model_output_path"] = optimization.output_graph response_data["size_input_model"] = get_size(optimization.input_graph) response_data["is_custom_dataloader"] = bool(workdir.template_path) workdir.update_data( request_id=request_id, model_path=optimization.input_graph, model_output_path=optimization.output_graph, metric=response_data, status="success", execution_details={"optimization": optimization.serialize()}, input_precision=optimization.input_precision, output_precision=optimization.output_precision, ) response_data["execution_details"] = {"optimization": optimization.serialize()} log.debug(f"Parsed data is {json.dumps(response_data)}") mq.post_success("optimization_finish", response_data) return response_data else: log.debug("FAIL") workdir.update_data( request_id=request_id, model_path=optimization.input_graph, input_precision=optimization.input_precision, output_precision=optimization.output_precision, status="error", ) mq.post_failure("optimization_finish", {"message": "failed", "id": request_id}) raise ClientErrorException("Optimization failed during execution.")
def load(self, path: str) -> None: """Load configuration from file.""" log.debug(f"Loading predefined config from {path}") with open(path) as yaml_config: config = yaml.safe_load(yaml_config) self.initialize(config)
def execute_benchmark(data: Dict[str, Any]) -> None: """ Execute benchmark. Expected data: { "id": "configuration_id", "workspace_path": "/path/to/workspace", "models": [ { "precision": "fp32", "path": "/localdisk/fp32.pb" }, { "precision": "int8", "path": "/localdisk/int8.pb" } ] } """ from lpot.ux.utils.workload.workload import Workload request_id = str(data.get("id", "")) models = data.get("models", None) if not (request_id and models): message = "Missing request id or model list." mq.post_error( "benchmark_finish", {"message": message, "code": 404, "id": request_id}, ) raise ClientErrorException(message) workdir = Workdir(request_id=request_id, overwrite=False) try: workload_path = workdir.workload_path workload_data = load_json( os.path.join(workload_path, "workload.json"), ) except Exception as err: mq.post_error( "benchmark_finish", {"message": repr(err), "code": 404, "id": request_id}, ) raise ClientErrorException(repr(err)) workload = Workload(workload_data) response_data: Dict[str, Any] = {"id": request_id, "execution_details": {}} mq.post_success( "benchmark_start", { "message": "started", "id": request_id, }, ) for idx, model_info in enumerate(models, start=1): model_precision = model_info.get("precision", None) model_path = model_info.get("path", None) benchmark_mode = model_info.get("mode", "performance") if not (model_precision and model_path): message = "Missing model precision or model path." mq.post_error( "benchmark_finish", {"message": message, "code": 404, "id": request_id}, ) raise ClientErrorException(message) benchmark: Benchmark = Benchmark( workload=workload, model_path=model_path, datatype=model_precision, mode=benchmark_mode, ) log_name = f"{model_precision}_{benchmark_mode}_benchmark" executor = Executor( workload_path, subject="benchmark", data={"id": request_id}, send_response=False, log_name=log_name, additional_log_names=["output.txt"], ) proc = executor.call( benchmark.command, ) logs = [os.path.join(workload_path, f"{log_name}.txt")] if proc.is_ok: parser = Parser(logs) metrics = parser.process() metric = {} execution_details: Dict[str, Any] = {} throughput_field = f"perf_throughput_{model_precision}" if isinstance(metrics, dict): metric = {throughput_field: metrics.get(throughput_field, "")} execution_details = { f"{model_precision}_benchmark": benchmark.serialize(), } response_data.update({"progress": f"{idx}/{len(models)}"}) response_data.update(metric) response_data["execution_details"].update(execution_details) workdir.update_metrics( request_id=request_id, metric_data=metric, ) workdir.update_execution_details( request_id=request_id, execution_details=execution_details, ) log.debug(f"Parsed data is {json.dumps(response_data)}") mq.post_success("benchmark_progress", response_data) else: log.error("Benchmark failed.") mq.post_failure("benchmark_finish", {"message": "failed", "id": request_id}) raise ClientErrorException("Benchmark failed during execution.") mq.post_success("benchmark_finish", response_data)