def load(self):
"""Start loading data from the latest summary file to the loader."""
self.status = _LoaderStatus.LOADING.value
filenames = []
for filename in FileHandler.list_dir(self._loader_info['summary_dir']):
if FileHandler.is_file(
FileHandler.join(self._loader_info['summary_dir'],
filename)):
filenames.append(filename)
filenames = ExplainLoader._filter_files(filenames)
if not filenames:
raise TrainJobNotExistError(
'No summary file found in %s, explain job will be delete.' %
self._loader_info['summary_dir'])
is_end = False
while not is_end and self.status != _LoaderStatus.STOP.value:
try:
file_changed, is_end, event_dict = self._parser.list_events(
filenames)
except UnknownError:
break
if file_changed:
logger.info(
'Summary file in %s update, reload the data in the summary.',
self._loader_info['summary_dir'])
self._clear_job()
if event_dict:
self._import_data_from_event(event_dict)
def load(self):
"""Start loading data from the latest summary file to the loader."""
filenames = []
for filename in FileHandler.list_dir(self._loader_info['summary_dir']):
if FileHandler.is_file(
FileHandler.join(self._loader_info['summary_dir'],
filename)):
filenames.append(filename)
filenames = ExplainLoader._filter_files(filenames)
if not filenames:
raise TrainJobNotExistError(
'No summary file found in %s, explain job will be delete.' %
self._loader_info['summary_dir'])
is_end = False
while not is_end:
is_clean, is_end, event_dict = self._parser.parse_explain(
filenames)
if is_clean:
logger.info(
'Summary file in %s update, reload the data in the summary.',
self._loader_info['summary_dir'])
self._clear_job()
if event_dict:
self._import_data_from_event(event_dict)
def filter_valid_files(self):
"""
Gets a list of valid files from the given file path.
Returns:
list[str], file name list.
"""
filenames = []
for filename in FileHandler.list_dir(self._summary_dir):
if FileHandler.is_file(FileHandler.join(self._summary_dir, filename)):
filenames.append(filename)
valid_filenames = []
for parser in self._parser_list:
valid_filenames.extend(parser.filter_files(filenames))
return list(set(valid_filenames))