def test_list_summaries(self):
"""Test list_summaries method success."""
summary_base_dir = tempfile.mkdtemp(dir=self.base_dir)
file_count = 10
directory_count = 1
gen_directories_and_files(summary_base_dir, file_count, directory_count)
summary_watcher = SummaryWatcher()
summaries = summary_watcher.list_summaries(summary_base_dir)
assert len(summaries) == file_count
summaries = summary_watcher.list_summaries(summary_base_dir, './\x00')
assert not summaries
shutil.rmtree(summary_base_dir)
def get_latest_lineage_summaries(summary_base_dir):
"""
Get all latest lineage summary logs in summary base dir.
Args:
summary_base_dir (str): Summary base dir.
Returns:
list[str], all latest lineage summary logs in summary base dir. The
lineage summary log is absolute path.
"""
summary_watcher = SummaryWatcher()
relative_dirs = summary_watcher.list_summary_directories(
summary_base_dir=summary_base_dir)
latest_summaries = []
for item in relative_dirs:
relative_dir = item.get('relative_path')
summaries = summary_watcher.list_summaries(
summary_base_dir=summary_base_dir, relative_path=relative_dir)
latest_file_name = SummaryPathParser._get_latest_lineage_file(
summaries)
if latest_file_name is None:
continue
latest_file = os.path.realpath(
os.path.join(summary_base_dir, relative_dir, latest_file_name))
latest_summaries.append(latest_file)
return latest_summaries
def get_lineage_summaries(summary_dir, is_sorted=False, reverse=True):
"""
Get lineage summaries according to summary dir.
Args:
summary_dir (str): Summary dir.
is_sorted (bool): If it is True, files will be sorted.
reverse (bool): If it is True, sort by timestamp increments and filename decrement.
Returns:
list, if the lineage summary log exist, return the file names, else return [].
"""
try:
summary_watcher = SummaryWatcher()
summaries = summary_watcher.list_summaries(
summary_base_dir=summary_dir)
except MindInsightException as err:
logger.warning(str(err))
return []
summary_files = [summary.get('file_name') for summary in summaries]
lineage_files_name = list(
filter(
lambda filename:
(filename.endswith(SummaryPathParser.LINEAGE_SUMMARY_SUFFIX)),
summary_files))
if is_sorted:
lineage_files_name = SummaryPathParser._sorted_summary_files(
lineage_files_name, reverse)
return lineage_files_name
def get_latest_lineage_summary(summary_dir):
"""
Get latest lineage summary log path according to summary dir.
Args:
summary_dir (str): Summary dir.
Returns:
Union[str, None], if the lineage summary log exist, return the path,
else return None. The lineage summary log path is absolute path.
"""
summary_watcher = SummaryWatcher()
summaries = summary_watcher.list_summaries(
summary_base_dir=summary_dir)
latest_file_name = SummaryPathParser._get_latest_lineage_file(
summaries)
return os.path.join(summary_dir, latest_file_name) \
if latest_file_name is not None else None