def parse_files(self, executor, filenames, events_data):
pb_filenames = self.filter_files(filenames)
pb_filenames = self.sort_files(pb_filenames)
for filename in pb_filenames:
if not self._set_latest_file(filename):
continue
future = executor.submit(self._parse_pb_file, self._summary_dir,
filename)
def add_tensor_event(future_value):
tensor_event = future_value.result()
if tensor_event is not None:
events_data.add_tensor_event(tensor_event)
future.add_done_callback(
exception_no_raise_wrapper(add_tensor_event))
return False
return True
def _load_single_file(self, file_handler, executor, events_data):
"""
Load a log file data.
Args:
file_handler (FileHandler): A file handler.
executor (Executor): The executor instance.
events_data (EventsData): The container of event data.
Returns:
bool, True if the summary file is finished loading.
"""
while True:
start_offset = file_handler.offset
try:
event_str = self.event_load(file_handler)
if event_str is None:
file_handler.reset_offset(start_offset)
return True
if len(event_str) > MAX_EVENT_STRING:
logger.warning(
"file_path: %s, event string: %d exceeds %d and drop it.",
file_handler.file_path, len(event_str),
MAX_EVENT_STRING)
continue
future = executor.submit(self._event_parse, event_str,
self._latest_filename)
def _add_tensor_event_callback(future_value):
tensor_values = future_value.result()
for tensor_value in tensor_values:
if tensor_value.plugin_name == PluginNameEnum.GRAPH.value:
try:
graph_tags = events_data.list_tags_by_plugin(
PluginNameEnum.GRAPH.value)
except KeyError:
graph_tags = []
summary_tags = self.filter_files(graph_tags)
for tag in summary_tags:
events_data.delete_tensor_event(tag)
events_data.add_tensor_event(tensor_value)
future.add_done_callback(
exception_no_raise_wrapper(_add_tensor_event_callback))
return False
except (exceptions.CRCFailedError,
exceptions.CRCLengthFailedError) as exc:
file_handler.reset_offset(start_offset)
file_size = file_handler.file_stat(file_handler.file_path).size
logger.error(
"Check crc failed and ignore this file, please check the integrity of the file, "
"file_path: %s, offset: %s, file size: %s. Detail: %s.",
file_handler.file_path, file_handler.offset, file_size,
str(exc))
return True
except (OSError, DecodeError,
exceptions.MindInsightException) as ex:
logger.error(
"Parse log file fail, and ignore this file, detail: %r, "
"file path: %s.", str(ex), file_handler.file_path)
return True
except Exception as ex:
logger.exception(ex)
raise UnknownError(str(ex))