def phase_pb_file(file_path: str) -> Union[MSGraph, None]:
"""
Parse pb file to graph
Args:
file_path (str): The file path of pb file.
Returns:
MSGraph, if load pb file and build graph success, will return the graph, else return None.
"""
if not CONFIG.VERBOSE:
logger.setLevel(logging.ERROR)
logger.info("Start to load graph from pb file, file path: %s.", file_path)
model_proto = anf_ir_pb2.ModelProto()
try:
model_proto.ParseFromString(FileHandler(file_path).read())
except ParseError:
logger.warning("The given file is not a valid pb file, file path: %s.",
file_path)
return None
graph = MSGraph()
try:
graph.build_graph(model_proto.graph)
except Exception as ex:
logger.error("Build graph failed, file path: %s.", file_path)
logger.exception(ex)
raise UnknownError(str(ex))
logger.info("Build graph success, file path: %s.", file_path)
return graph
def _load_data_in_thread(self):
"""Log (but not swallow) exceptions in thread to help debugging."""
try:
self._load_data()
except Exception as exc:
logger.exception(exc)
raise UnknownError('Load data thread error.')
def _load_single_file(self, file_handler):
"""
Load a log file data.
Args:
file_handler (FileHandler): A file handler.
"""
logger.debug("Load single summary file, file path: %s.", file_handler.file_path)
while True:
start_offset = file_handler.offset
try:
event_str = self._event_load(file_handler)
if event_str is None:
file_handler.reset_offset(start_offset)
break
event = summary_pb2.Event.FromString(event_str)
self._event_parse(event)
except exceptions.CRCFailedError:
file_handler.reset_offset(start_offset)
logger.warning("Check crc faild and ignore this file, file_path=%s, "
"offset=%s.", file_handler.file_path, file_handler.offset)
break
except (OSError, DecodeError, exceptions.MindInsightException) as ex:
logger.warning("Parse log file fail, and ignore this file, detail: %r,"
"file path: %s.", str(ex), file_handler.file_path)
break
except Exception as ex:
logger.exception(ex)
raise UnknownError(str(ex))
def _load_single_file(self, file_handler, executor):
"""
Load a log file data.
Args:
file_handler (FileHandler): A file handler.
executor (Executor): The executor instance.
Returns:
bool, True if the summary file is finished loading.
"""
while True:
start_offset = file_handler.offset
try:
event_str = self._event_load(file_handler)
if event_str is None:
file_handler.reset_offset(start_offset)
return True
if len(event_str) > MAX_EVENT_STRING:
logger.warning("file_path: %s, event string: %d exceeds %d and drop it.",
file_handler.file_path, len(event_str), MAX_EVENT_STRING)
continue
future = executor.submit(self._event_parse, event_str, self._latest_filename)
def _add_tensor_event_callback(future_value):
try:
tensor_values = future_value.result()
for tensor_value in tensor_values:
if tensor_value.plugin_name == PluginNameEnum.GRAPH.value:
try:
graph_tags = self._events_data.list_tags_by_plugin(PluginNameEnum.GRAPH.value)
except KeyError:
graph_tags = []
summary_tags = self.filter_files(graph_tags)
for tag in summary_tags:
self._events_data.delete_tensor_event(tag)
self._events_data.add_tensor_event(tensor_value)
except Exception as exc:
# Log exception for debugging.
logger.exception(exc)
raise
future.add_done_callback(_add_tensor_event_callback)
return False
except exceptions.CRCFailedError:
file_handler.reset_offset(start_offset)
logger.warning("Check crc faild and ignore this file, file_path=%s, "
"offset=%s.", file_handler.file_path, file_handler.offset)
return True
except (OSError, DecodeError, exceptions.MindInsightException) as ex:
logger.warning("Parse log file fail, and ignore this file, detail: %r,"
"file path: %s.", str(ex), file_handler.file_path)
return True
except Exception as ex:
logger.exception(ex)
raise UnknownError(str(ex))
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except (PathNotDirectoryError, FileNotFoundError) as err:
# except PathNotDirectoryError and FileNotFoundError as they are on warning level
logger.warning(str(err))
except Exception as exc:
logger.exception(exc)
raise UnknownError(str(exc))
def handle_unknown_error(ex):
"""Handle unknown error."""
logger.error('%r %r detail: %r', request.method, quote(request.path),
str(ex))
logger.exception(ex)
if isinstance(ex, PermissionError):
error = FileSystemPermissionError('File System Permission Error')
else:
error = UnknownError('System error.')
res_body = dict(error_code=error.error_code, error_msg=error.message)
return jsonify(res_body), error.http_code
def handle_http_exception_error(ex):
"""Handle http exception error."""
logger.warning('%r %r, detail: %r', request.method, quote(request.path),
str(ex))
if isinstance(ex, NotFound):
error = RestfulApiNotExist()
elif isinstance(ex, MethodNotAllowed):
error = RequestMethodNotAllowed()
else:
logger.exception(ex)
error = UnknownError('System error or http error.')
res_body = {"error_code": error.error_code, "error_msg": error.message}
return jsonify(res_body), error.http_code
def _make_summary_base_dir(self, summary_base_dir):
"""Check and make summary_base_dir."""
if not os.path.exists(summary_base_dir):
permissions = os.R_OK | os.W_OK | os.X_OK
os.umask(permissions << 3 | permissions)
mode = permissions << 6
try:
logger.info(
"The summary_base_dir is generated automatically, path is %s.",
summary_base_dir)
os.makedirs(summary_base_dir, mode=mode, exist_ok=True)
except OSError as exc:
raise UnknownError(
"Can not make the summary base directory. Detail: %s." %
str(exc))
def _validate_config(self, config_path):
"""Check config_path."""
config_path = self._normalize_path("config_path", config_path)
try:
with open(config_path, "r") as file:
config_info = yaml.safe_load(file)
except PermissionError as exc:
raise FileSystemPermissionError(
"Can not open config file. Detail: %s." % str(exc))
except Exception as exc:
raise UnknownError("Detail: %s." % str(exc))
self._validate_config_schema(config_info)
config_info['summary_base_dir'] = self._normalize_path(
"summary_base_dir", config_info.get('summary_base_dir'))
self._make_summary_base_dir(config_info['summary_base_dir'])
return config_info
def _parse_pb_file(summary_dir, filename):
"""
Parse pb file and write content to `EventsData`.
Args:
filename (str): The file path of pb file.
Returns:
TensorEvent, if load pb file and build graph success, will return tensor event, else return None.
"""
file_path = FileHandler.join(summary_dir, filename)
logger.info("Start to load graph from pb file, file path: %s.",
file_path)
filehandler = FileHandler(file_path)
model_proto = anf_ir_pb2.ModelProto()
try:
model_proto.ParseFromString(filehandler.read())
except ParseError:
logger.warning(
"The given file is not a valid pb file, file path: %s.",
file_path)
return None
graph = MSGraph()
try:
graph.build_graph(model_proto.graph)
except Exception as ex:
# Normally, there are no exceptions, and it is only possible for users on the MindSpore side
# to dump other non-default graphs.
logger.error("Build graph failed, file path: %s.", file_path)
logger.exception(ex)
raise UnknownError(str(ex))
tensor_event = TensorEvent(
wall_time=FileHandler.file_stat(file_path).mtime,
step=0,
tag=filename,
plugin_name=PluginNameEnum.GRAPH.value,
value=graph,
filename=filename)
logger.info("Build graph success, file path: %s.", file_path)
return tensor_event
def _get_hoc_image(self, image_path, train_id):
"""Get hoc image for image data demonstration in UI."""
sample_id, label, layer = image_path.strip(".jpg").split("_")
layer = int(layer)
job = self.job_manager.get_job(train_id)
samples = job.samples
label_idx = job.labels.index(label)
chosen_sample = samples[int(sample_id)]
original_path_image = chosen_sample['image']
abs_image_path = os.path.join(self.job_manager.summary_base_dir,
_clean_train_id_b4_join(train_id),
original_path_image)
if self._is_forbidden(abs_image_path):
raise FileSystemPermissionError("Forbidden.")
image_type = ImageQueryTypes.OUTCOME.value
try:
image = Image.open(abs_image_path)
except FileNotFoundError:
raise ImageNotExistError(
f"train_id:{train_id} path:{image_path} type:{image_type}")
except PermissionError:
raise FileSystemPermissionError(
f"train_id:{train_id} path:{image_path} type:{image_type}")
except OSError:
raise UnknownError(
f"Invalid image file: train_id:{train_id} path:{image_path} type:{image_type}"
)
edit_steps = []
boxes = chosen_sample["hierarchical_occlusion"][label_idx][
"hoc_layers"][layer]["boxes"]
mask = chosen_sample["hierarchical_occlusion"][label_idx]["mask"]
for box in boxes:
edit_steps.append(EditStep(layer, *box))
image_cp = pil_apply_edit_steps(image, mask, edit_steps)
buffer = io.BytesIO()
image_cp.save(buffer, format=_PNG_FORMAT)
return buffer.getvalue()
def run(self, args):
"""
Execute for start command.
Args:
args (Namespace): Parsed arguments to hold customized parameters.
"""
try:
date_time = datetime.datetime.now().strftime(
'output_%Y%m%d_%H%M%S_%f')
output_path = os.path.join(args.output, date_time)
summary_dir = args.summary_dir
if not self._check_dirpath(summary_dir):
return
summary_parser = _SummaryParser(summary_dir)
summary_files = summary_parser.filter_files(
os.listdir(summary_dir))
if not summary_files:
parse_summary_logger.error('Path %s has no summary file.',
summary_dir)
return
summary_files = summary_parser.sort_files(summary_files)
filename = summary_files[-1]
summary_file = FileHandler.join(summary_dir, filename)
if not (self._check_filepath(summary_file)
and self._check_create_filepath(output_path)
and self._check_create_filepath(
FileHandler.join(output_path, 'image'))):
return
eventparser = EventParser(summary_file, output_path)
eventparser.parse()
except Exception as ex:
parse_summary_logger.error(
"Parse summary file failed, detail: %r.", str(ex))
raise UnknownError(str(ex))
def _load_data(self):
"""
Prepare loaders in cache and start loading the data from summaries.
Only a limited number of loaders will be cached in terms of updated_time or query_time. The size of cache
pool is determined by _MAX_LOADERS_NUM. When the manager start loading data, only the latest _MAX_LOADER_NUM
summaries will be loaded in cache. If a cached loader if queries by 'get_job', the query_time of the loader
will be updated as well as the the loader moved to the end of cache. If an uncached summary is queried,
a new loader instance will be generated and put to the end cache.
"""
try:
with self._load_data_mutex:
if self.status == _ExplainManagerStatus.LOADING.value:
logger.info(
'Current status is %s, will ignore to load data.',
self.status)
return
logger.info('Start to load data, and status change to %s.',
_ExplainManagerStatus.LOADING.value)
self.status = _ExplainManagerStatus.LOADING.value
self._cache_loaders()
if self.status == _ExplainManagerStatus.STOPPING.value:
logger.info(
'The manager status has been %s, will not execute loading.',
self.status)
return
self._execute_loading()
logger.info(
'Load event data end, current status: %s, next status: %s, loader pool size: %d.',
self.status, _ExplainManagerStatus.DONE.value,
len(self._loader_pool))
except Exception as ex:
logger.exception(ex)
raise UnknownError(str(ex))
finally:
self.status = _ExplainManagerStatus.DONE.value
def wrapper(*args, **kwargs):
try:
func(*args, **kwargs)
except Exception as exc:
logger.exception(exc)
raise UnknownError(str(exc))
def list_events(self, filenames):
"""
Load summary file and parse file content.
Args:
filenames (list[str]): File name list.
Returns:
tuple, the elements of the tuple are:
- file_changed (bool): True if the latest file is changed.
- is_end (bool): True if all the summary files are finished loading.
- event_data (dict): Event data where keys are explanation field.
"""
summary_files = self.sort_files(filenames)
is_end = False
file_changed = False
event_data = {}
filename = summary_files[-1]
file_path = FileHandler.join(self._summary_dir, filename)
if filename != self._latest_filename:
self._summary_file_handler = FileHandler(file_path, 'rb')
self._latest_filename = filename
self._latest_offset = 0
file_changed = True
new_size = FileHandler.file_stat(file_path).size
if new_size == self._latest_offset:
is_end = True
return file_changed, is_end, event_data
while True:
start_offset = self._summary_file_handler.offset
try:
event_str = self.event_load(self._summary_file_handler)
if event_str is None:
self._summary_file_handler.reset_offset(start_offset)
is_end = True
return file_changed, is_end, event_data
if len(event_str) > MAX_EVENT_STRING:
logger.warning(
"file_path: %s, event string: %d exceeds %d and drop it.",
self._summary_file_handler.file_path, len(event_str),
MAX_EVENT_STRING)
continue
field_list, tensor_value_list = self._event_decode(event_str)
for field, tensor_value in zip(field_list, tensor_value_list):
event_data[field] = tensor_value
logger.debug("Parse summary file offset %d, file path: %s.",
self._summary_file_handler.offset, file_path)
return file_changed, is_end, event_data
except exceptions.CRCLengthFailedError as ex:
self._summary_file_handler.reset_offset(start_offset)
is_end = True
logger.warning(
"Check crc failed and reset offset, file_path=%s, offset=%s. Detail: %r.",
self._summary_file_handler.file_path,
self._summary_file_handler.offset, str(ex))
return file_changed, is_end, event_data
except Exception as ex:
# Note: If an unknown error occurs, we will set the offset to the end of this file,
# which is equivalent to stopping parsing this file. We do not delete the current job
# and retain the data that has been successfully parsed.
self._summary_file_handler.reset_offset(new_size)
# Notice: If the current job is the latest one in the loader pool and the job is deleted,
# the job goes into an infinite cycle of load-fail-delete-reload-load-fail-delete.
# We need to prevent this infinite loop.
logger.error(
"Parse summary file failed, will set offset to the file end. file_path: %s, "
"offset: %d, detail: %s.", file_path,
self._summary_file_handler.offset, str(ex))
logger.exception(ex)
raise UnknownError(str(ex))
finally:
self._latest_offset = self._summary_file_handler.offset
def parse_explain(self, filenames):
"""
Load summary file and parse file content.
Args:
filenames (list[str]): File name list.
Returns:
bool, True if all the summary files are finished loading.
"""
summary_files = self.sort_files(filenames)
is_end = False
is_clean = False
event_data = {}
filename = summary_files[-1]
file_path = FileHandler.join(self._summary_dir, filename)
if filename != self._latest_filename:
self._summary_file_handler = FileHandler(file_path, 'rb')
self._latest_filename = filename
self._latest_file_size = 0
is_clean = True
new_size = FileHandler.file_stat(file_path).size
if new_size == self._latest_file_size:
is_end = True
return is_clean, is_end, event_data
while True:
start_offset = self._summary_file_handler.offset
try:
event_str = self.event_load(self._summary_file_handler)
if event_str is None:
self._summary_file_handler.reset_offset(start_offset)
is_end = True
return is_clean, is_end, event_data
if len(event_str) > MAX_EVENT_STRING:
logger.warning(
"file_path: %s, event string: %d exceeds %d and drop it.",
self._summary_file_handler.file_path, len(event_str),
MAX_EVENT_STRING)
continue
field_list, tensor_value_list = self._event_decode(event_str)
for field, tensor_value in zip(field_list, tensor_value_list):
event_data[field] = tensor_value
logger.info("Parse summary file offset %d, file path: %s.",
self._summary_file_handler.offset, file_path)
return is_clean, is_end, event_data
except (exceptions.CRCFailedError,
exceptions.CRCLengthFailedError) as ex:
self._summary_file_handler.reset_offset(start_offset)
is_end = True
logger.warning(
"Check crc failed and ignore this file, file_path=%s, offset=%s. Detail: %r.",
self._summary_file_handler.file_path,
self._summary_file_handler.offset, str(ex))
return is_clean, is_end, event_data
except (OSError, DecodeError,
exceptions.MindInsightException) as ex:
is_end = True
logger.warning(
"Parse log file fail, and ignore this file, detail: %r,"
"file path: %s.", str(ex),
self._summary_file_handler.file_path)
return is_clean, is_end, event_data
except Exception as ex:
logger.exception(ex)
raise UnknownError(str(ex))
def query_image_binary(self, train_id, image_path, image_type):
"""
Query image binary content.
Args:
train_id (str): Job ID.
image_path (str): Image path relative to explain job's summary directory.
image_type (str): Image type, 'original' or 'overlay'.
Returns:
bytes, image binary.
"""
abs_image_path = os.path.join(self.job_manager.summary_base_dir,
_clean_train_id_b4_join(train_id),
image_path)
if self._is_forbidden(abs_image_path):
raise FileSystemPermissionError("Forbidden.")
try:
if image_type != "overlay":
# no need to convert
with open(abs_image_path, "rb") as fp:
return fp.read()
image = Image.open(abs_image_path)
if image.mode == _RGBA_MODE:
# It is RGBA already, do not convert.
with open(abs_image_path, "rb") as fp:
return fp.read()
except FileNotFoundError:
raise ImageNotExistError(
f"train_id:{train_id} path:{image_path} type:{image_type}")
except PermissionError:
raise FileSystemPermissionError(
f"train_id:{train_id} path:{image_path} type:{image_type}")
except OSError:
raise UnknownError(
f"Invalid image file: train_id:{train_id} path:{image_path} type:{image_type}"
)
if image.mode == _SINGLE_CHANNEL_MODE:
saliency = np.asarray(image) / _UINT8_MAX
elif image.mode == _RGB_MODE:
saliency = np.asarray(image)
saliency = saliency[:, :, 0] / _UINT8_MAX
else:
raise UnknownError(f"Invalid overlay image mode:{image.mode}.")
saliency_stack = np.empty((saliency.shape[0], saliency.shape[1], 4))
for c in range(3):
saliency_stack[:, :, c] = saliency
rgba = saliency_stack * _SALIENCY_CMAP_HI
rgba += (1 - saliency_stack) * _SALIENCY_CMAP_LOW
rgba[:, :, 3] = saliency * _UINT8_MAX
overlay = Image.fromarray(np.uint8(rgba), mode=_RGBA_MODE)
buffer = io.BytesIO()
overlay.save(buffer, format=_PNG_FORMAT)
return buffer.getvalue()