def __init__(
self,
workspace_path: str,
subject: str,
data: Optional[dict] = None,
send_response: bool = True,
log_name: Optional[str] = None,
additional_log_names: List[str] = [],
) -> None:
"""
Bash Executor constructor.
:param args: arguments for parser execution
"""
if data:
self._request_id = str(data.get("id", ""))
self._log_name = log_name
self._additional_log_names = additional_log_names
self._workdir = workspace_path
self._mq = MessageQueue()
self._subject = subject
self._send_response = send_response
self.data = data
self.time_start: Optional[datetime.datetime] = None
self.time_stop: Optional[datetime.datetime] = None
def __init__(self, data: Dict[str, Any]) -> None:
"""Initialize Downloader class."""
self.request_id: str = str(data.get("id", ""))
self.framework: str = data.get("framework", "")
self.domain: str = data.get("domain", "")
self.model: str = data.get("model", "")
self.workspace_path: str = data.get("workspace_path", "")
self.progress_steps: Optional[int] = data.get("progress_steps", None)
self.download_dir: str = ""
self.mq = MessageQueue()
def web_socket_publisher(web_socket: SocketIO) -> None:
"""Send messages from queue via web-socket to GUI."""
queue = MessageQueue()
while True:
message = queue.get()
web_socket.emit(
message.subject,
{"status": message.status, "data": message.data},
broadcast=True,
)
class Downloader:
"""UX model downloader class."""
def __init__(self, data: Dict[str, Any]) -> None:
"""Initialize Downloader class."""
self.request_id: str = str(data.get("id", ""))
self.framework: str = data.get("framework", "")
self.domain: str = data.get("domain", "")
self.model: str = data.get("model", "")
self.workspace_path: str = data.get("workspace_path", "")
self.progress_steps: Optional[int] = data.get("progress_steps", None)
self.download_dir: str = ""
self.mq = MessageQueue()
def download_config(self) -> None:
"""Find yaml config resource and initialize downloading."""
if not (self.request_id and self.framework and self.domain
and self.model and self.workspace_path):
message = "Missing request id, workspace path, framework, domain or model."
self.mq.post_error(
"download_finish",
{
"message": message,
"code": 404,
"id": self.request_id
},
)
raise ClientErrorException(message)
model_config = load_model_config()
model_info = (model_config.get(self.framework,
{}).get(self.domain,
{}).get(self.model, None))
if model_info is None:
raise Exception(
f"{self.framework} {self.domain} {self.model} is not supported.",
)
self.download_dir = os.path.join(
self.workspace_path,
"examples",
self.framework,
self.domain,
self.model,
)
self.download_yaml_config(model_info)
def download_yaml_config(self, model_info: Dict[str, Any]) -> None:
"""Download config from GitHub for specified model."""
yaml_relative_location = model_info.get("yaml", "")
if not yaml_relative_location:
message = "Missing yaml location."
self.mq.post_error(
"download_finish",
{
"message": message,
"code": 404,
"id": self.request_id
},
)
raise ClientErrorException(message)
url, headers = self.get_yaml_url(yaml_relative_location, )
download_path = os.path.join(
self.download_dir,
os.path.basename(yaml_relative_location),
)
self.download_file(
url=url,
headers=headers,
download_path=download_path,
)
self.mq.post_success(
"download_finish",
{
"id": self.request_id,
"path": download_path,
},
)
def download_model(self) -> None:
"""Find model resource and initialize downloading."""
model_config = load_model_config()
model_info = (model_config.get(self.framework,
{}).get(self.domain,
{}).get(self.model, None))
if model_info is None:
raise Exception(
f"{self.framework} {self.domain} {self.model} is not supported.",
)
self.download_dir = os.path.join(
self.workspace_path,
"examples",
self.framework,
self.domain,
self.model,
)
self.download(model_info)
def download(self, model_info: Dict[str, Any]) -> None:
"""Download specified model."""
download_info = model_info.get("download", None)
if download_info is None:
message = "Model download is not supported."
self.mq.post_error(
"download_finish",
{
"message": message,
"code": 404,
"id": self.request_id
},
)
raise ClientErrorException(message)
url = download_info.get("url")
filename = download_info.get("filename")
is_archived = download_info.get("is_archived")
if not (url and filename):
message = "Could not found download link for model or output file name."
self.mq.post_error(
"download_finish",
{
"message": message,
"code": 404,
"id": self.request_id
},
)
raise ClientErrorException(message)
download_path = os.path.join(self.download_dir, filename)
if is_archived:
download_path = os.path.join(self.download_dir, url.split("/")[-1])
self.download_file(
url=url,
download_path=download_path,
)
model_path = download_path
if is_archived:
model_path = self.unpack_archive(download_path, filename)
self.mq.post_success(
"download_finish",
{
"id": self.request_id,
"path": model_path,
},
)
def download_file(
self,
url: str,
download_path: str,
headers: Optional[dict] = {},
) -> None:
"""Download specified file."""
try:
with requests.get(
url,
allow_redirects=True,
stream=True,
headers=headers,
) as r:
r.raise_for_status()
os.makedirs(os.path.dirname(download_path), exist_ok=True)
with open(download_path, "wb") as f:
log.debug(f"Download file from {url} to {download_path}")
total_length = r.headers.get("content-length")
self.mq.post_success(
"download_start",
{
"message": "started",
"id": self.request_id,
"url": url,
},
)
if total_length is None:
f.write(r.content)
return
downloaded = 0
last_progress = 0
total_size = int(total_length)
for data in r.iter_content(chunk_size=4096):
downloaded += len(data)
f.write(data)
if self.progress_steps:
progress = int(100 * downloaded / total_size)
if (last_progress != progress and progress %
int(100 / self.progress_steps) == 0):
self.mq.post_success(
"download_progress",
{
"id": self.request_id,
"progress":
f"{downloaded}/{total_size}",
},
)
log.debug(f"Download progress: {progress}%")
last_progress = progress
except requests.exceptions.HTTPError:
message = f"Error downloading file from {url} to {download_path}"
self.mq.post_error(
"download_finish",
{
"message": message,
"code": 404,
"id": self.request_id,
},
)
return
def unpack_archive(self, archive_path: str, filename: str) -> str:
"""Unpack archive and return path to unpacked model."""
self.mq.post_success(
"unpack_start",
{
"id": self.request_id,
},
)
log.debug(f"Unpacking {archive_path}")
if zipfile.is_zipfile(archive_path):
z = zipfile.ZipFile(archive_path)
z.extractall(self.download_dir)
elif tarfile.is_tarfile(archive_path):
t = tarfile.open(archive_path, "r:gz")
t.extractall(self.download_dir)
else:
message = (
"Could unpack an archive. Supported archive types are zip and tar.gz."
)
self.mq.post_error(
"unpack_finish",
{
"message": message,
"code": 404,
"id": self.request_id
},
)
raise ClientErrorException(message)
os.remove(archive_path)
unpacked_path = os.path.join(self.download_dir, filename)
self.mq.post_success(
"unpack_finish",
{
"id": self.request_id,
"path": unpacked_path
},
)
log.debug(f"Model file has been extracted to {unpacked_path}")
return unpacked_path
def get_yaml_url(
self,
yaml_relative_location: str,
) -> Tuple[str, dict]:
"""Get url for yaml config download."""
if is_development_env():
from urllib.parse import quote_plus
file_path = quote_plus(
os.path.join(
"examples",
self.framework,
self.domain,
yaml_relative_location,
), )
url = os.path.join(
os.environ["LPOT_PROJECT_URL"],
file_path,
"raw?ref=developer",
)
headers = {"Private-Token": os.environ.get("LPOT_TOKEN")}
return url, headers
user = github_info.get("user")
repository = github_info.get("repository")
tag = github_info.get("tag")
if not (user, repository, tag):
message = "Missing github repository information."
self.mq.post_error(
"download_finish",
{
"message": message,
"code": 500,
"id": self.request_id
},
)
raise ClientErrorException(message)
url_prefix = f"https://raw.githubusercontent.com/{user}/{repository}/{tag}/"
url = os.path.join(
url_prefix,
"examples",
self.framework,
self.domain,
yaml_relative_location,
)
return url, {}
class Executor:
"""Executor class provide execute shell command."""
def __init__(
self,
workspace_path: str,
subject: str,
data: Optional[dict] = None,
send_response: bool = True,
log_name: Optional[str] = None,
additional_log_names: List[str] = [],
) -> None:
"""
Bash Executor constructor.
:param args: arguments for parser execution
"""
if data:
self._request_id = str(data.get("id", ""))
self._log_name = log_name
self._additional_log_names = additional_log_names
self._workdir = workspace_path
self._mq = MessageQueue()
self._subject = subject
self._send_response = send_response
self.data = data
self.time_start: Optional[datetime.datetime] = None
self.time_stop: Optional[datetime.datetime] = None
def call_one(
self,
args: List[Any],
logger: Optional[Any] = None,
executable: Optional[Any] = None,
shell: bool = False,
cwd: Optional[str] = None,
env: Optional[dict] = None,
universal_newlines: bool = False,
startupinfo: Optional[Any] = None,
creationflags: int = 0,
processes: Optional[Any] = None,
ignore_exit_codes: Union[list, Any] = None,
pid: Optional[str] = None,
) -> None:
"""
Execute single call for process.
:param args:
:param logger:
:param executable:
:param shell:
:param cwd:
:param env:
:param universal_newlines:
:param startupinfo:
:param creationflags:
:param processes:
:param ignore_exit_codes:
:param pid:
"""
proc = None
try:
log.debug("Exec %s ", " ".join(map(str, args)))
proc = Proc(
self.workdir,
pid=pid,
request_id=self.request_id,
filename=self.log_name,
additional_log_names=self.additional_log_names,
)
if self._send_response:
self._mq.post_success(
"_".join([self._subject, "start"]),
self.data,
)
proc.run(
args,
executable,
shell,
cwd,
env,
universal_newlines,
startupinfo,
creationflags,
ignore_exit_codes,
)
self.time_start = proc.time_start
self.time_stop = proc.time_stop
except Exception:
if self._send_response:
self._mq.post_error(
self._subject,
{
"message": Exception,
"id": self.request_id
},
)
log.exception("Unexpected error for command line %s", args)
try:
LOCK.acquire()
if processes is None:
processes = []
processes.append(proc)
finally:
LOCK.release()
log.debug("DONE")
def call(
self,
args: List[Any], # type: ignore
logger: Optional[Any] = None,
executable: Optional[Any] = None,
shell: bool = False,
cwd: Optional[str] = None,
env: Optional[dict] = None,
universal_newlines: bool = False,
startupinfo: Optional[Any] = None,
creationflags: int = 0,
env_args: Optional[list] = None,
ignore_exit_codes: Union[list, Any] = None,
pid: Optional[str] = None,
) -> LPOTProcesses:
"""
Execute multiple calls for process.
:param args:
:param logger:
:param executable:
:param shell:
:param cwd:
:param env:
:param universal_newlines:
:param startupinfo:
:param creationflags:
:param ignore_exit_codes:
:param pid:
"""
# update output directory to current stage
self.refresh_workdir()
threads = []
processes = LPOTProcesses()
if not self.is_multi_commands(args):
args = [args]
for arg in args:
threads.append(
Thread(
target=self.call_one,
args=(
arg,
logger,
executable,
shell,
cwd,
env_args,
universal_newlines,
startupinfo,
creationflags,
processes,
ignore_exit_codes,
pid,
),
daemon=True,
), )
# Start all threads
for command_thread in threads:
command_thread.start()
# Wait for all of them to finish
for command_thread in threads:
command_thread.join()
return processes
@property
def process_duration(self) -> Optional[float]:
"""Return duration of the process in [s]."""
if self.time_start and self.time_stop:
duration = self.time_stop - self.time_start
return duration.total_seconds()
return None
@property
def workdir(self) -> str:
"""
Property concat workdir path.
:return: workdir path
"""
return self._workdir
@property
def request_id(self) -> Optional[str]:
"""
Property contain info about request.
:return: request id
"""
return self._request_id
@property
def log_name(self) -> Optional[str]:
"""
Property contain info about output file name.
:return: requested log filename
"""
return self._log_name
@property
def additional_log_names(self) -> List[str]:
"""
Property contain info about additional output file names.
:return: list of additional log filenames
"""
return self._additional_log_names
def refresh_workdir(self) -> str:
"""
Property returns workdir method.
:return: workdir path
"""
return self.workdir
@staticmethod
def is_multi_commands(args: list) -> bool:
"""
Check type of execution.
:param args: arguments for parser execution
:return: bool value True if args are in list else False
"""
for arg in args:
if not isinstance(arg, list):
return False
# all elements must be lists
return True
import json
import os
from typing import Any, Dict, List
from lpot.ux.components.benchmark import Benchmarks
from lpot.ux.components.benchmark.benchmark import Benchmark
from lpot.ux.utils.exceptions import ClientErrorException, InternalException
from lpot.ux.utils.executor import Executor
from lpot.ux.utils.logger import log
from lpot.ux.utils.parser import BenchmarkParserFactory
from lpot.ux.utils.templates.workdir import Workdir
from lpot.ux.utils.utils import _load_json_as_dict
from lpot.ux.utils.workload.workload import Workload
from lpot.ux.web.communication import MessageQueue
mq = MessageQueue()
def execute_benchmark(data: Dict[str, Any]) -> None:
"""
Execute benchmark.
Expected data:
{
"id": "configuration_id",
"workspace_path": "/path/to/workspace",
"input_model": {
"precision": "fp32",
"path": "/localdisk/fp32.pb"
},
"optimized_model": {
def setUp(self) -> None:
"""Create test environment."""
self.queue = MessageQueue()
class TestCommunication(unittest.TestCase):
"""UX Communication tests."""
def setUp(self) -> None:
"""Create test environment."""
self.queue = MessageQueue()
def test_request(self) -> None:
"""Test that Request is working."""
method = "GET"
operation = "/api/a/b/x"
data = self._get_random_dict()
request = Request(method, operation, data)
self.assertEqual(method, request.method)
self.assertEqual(operation, request.operation)
self.assertEqual(data, request.data)
def test_response(self) -> None:
"""Test that Response is working."""
response = Response()
self.assertEqual({}, response.data)
self.assertEqual({}, response.command)
def test_create_simple_response(self) -> None:
"""Test that create_simple_response is working."""
data = self._get_random_dict()
response = create_simple_response(data)
self.assertEqual(data, response.data)
self.assertEqual({}, response.command)
def test_message(self) -> None:
"""Test that Message is working."""
status = "Test status"
subject = "Test subject"
data = self._get_random_dict()
message = Message(status, subject, data)
self.assertEqual(status, message.status)
self.assertEqual(subject, message.subject)
self.assertEqual(data, message.data)
def test_message_queue_post_failure(self) -> None:
"""Test posting failure messages to message queue."""
data = self._get_random_dict()
self.queue.post_failure("subject", data)
self._assert_message("failure", "subject", data)
def test_message_queue_post_success(self) -> None:
"""Test posting success messages to message queue."""
data = self._get_random_dict()
self.queue.post_success("subject", data)
self._assert_message("success", "subject", data)
def test_message_queue_post_error(self) -> None:
"""Test posting error messages to message queue."""
data = self._get_random_dict()
self.queue.post_error("subject", data)
self._assert_message("error", "subject", data)
def _get_random_dict(self, size: int = 5) -> dict:
"""Build random dict."""
from numpy.random import randint
return {"key " + str(i): randint(65536) for i in range(size)}
def _assert_message(
self,
expected_status: str,
expected_subject: str,
expected_data: dict,
) -> None:
"""Assert message in queue matches expectations."""
message = self.queue.get()
self.assertEqual(expected_status, message.status)
self.assertEqual(expected_subject, message.subject)
self.assertEqual(expected_data, message.data)