def _organize_from_disk(self):
"""Organize lineage objs from disk."""
if self._summary_base_dir is None:
return
summary_watcher = SummaryWatcher()
relative_dirs = summary_watcher.list_summary_directories(
summary_base_dir=self._summary_base_dir)
no_lineage_count = 0
for item in relative_dirs:
relative_dir = item.get('relative_path')
update_time = item.get('update_time')
abs_summary_dir = os.path.realpath(
os.path.join(self._summary_base_dir, relative_dir))
try:
lineage_parser = LineageParser(abs_summary_dir, update_time)
super_lineage_obj = lineage_parser.super_lineage_obj
if super_lineage_obj is not None:
self._super_lineage_objs.update(
{abs_summary_dir: super_lineage_obj})
except LineageFileNotFoundError:
no_lineage_count += 1
if no_lineage_count == len(relative_dirs):
logger.info('There is no summary log file under summary_base_dir.')
raise LineageFileNotFoundError(
'There is no summary log file under summary_base_dir.')
def get_latest_lineage_summaries(summary_base_dir):
"""
Get all latest lineage summary logs in summary base dir.
Args:
summary_base_dir (str): Summary base dir.
Returns:
list[str], all latest lineage summary logs in summary base dir. The
lineage summary log is absolute path.
"""
summary_watcher = SummaryWatcher()
relative_dirs = summary_watcher.list_summary_directories(
summary_base_dir=summary_base_dir)
latest_summaries = []
for item in relative_dirs:
relative_dir = item.get('relative_path')
summaries = summary_watcher.list_summaries(
summary_base_dir=summary_base_dir, relative_path=relative_dir)
latest_file_name = SummaryPathParser._get_latest_lineage_file(
summaries)
if latest_file_name is None:
continue
latest_file = os.path.realpath(
os.path.join(summary_base_dir, relative_dir, latest_file_name))
latest_summaries.append(latest_file)
return latest_summaries
def test_list_summary_directories_with_overall_on(self):
"""Test list_summary_directories method success."""
summary_base_dir = tempfile.mkdtemp(dir=self.base_dir)
file_count = 10
directory_count = 10
gen_directories_and_files(summary_base_dir, file_count, directory_count)
summary_watcher = SummaryWatcher()
directories = summary_watcher.list_summary_directories(summary_base_dir, overall=True)
expected_directory_count = directory_count + 1
assert len(directories) == min(expected_directory_count, SummaryWatcher.MAX_SUMMARY_DIR_COUNT)
shutil.rmtree(summary_base_dir)
def get_summary_dirs(summary_base_dir):
"""
Get summary dirs according to summary base dir.
Args:
summary_base_dir (str): Summary base dir.
Returns:
list[str], all summary dirs in summary base dir. The summary dir is
absolute path.
"""
summary_watcher = SummaryWatcher()
relative_dirs = summary_watcher.list_summary_directories(
summary_base_dir=summary_base_dir)
summary_dirs = list(
map(
lambda item: os.path.realpath(
os.path.join(summary_base_dir, item.get('relative_path'))),
relative_dirs))
return summary_dirs
class _BriefCacheManager(_BaseCacheManager):
"""A cache manager that holds all disk train jobs on disk."""
def __init__(self, summary_base_dir):
super(_BriefCacheManager, self).__init__(summary_base_dir)
self._summary_watcher = SummaryWatcher()
def cache_train_job(self, train_id):
"""
Cache given train job.
All disk train jobs are cached on every reload, so this method always return false.
Args:
train_id (str): Train Id.
"""
if train_id in self._cache_items:
self._cache_items[train_id].update_access_time()
return False
def update_cache(self, executor):
"""Update cache."""
logger.info('Start to update BriefCacheManager.')
summaries_info = self._summary_watcher.list_summary_directories(
self._summary_base_dir)
basic_train_jobs = []
for info in summaries_info:
basic_train_jobs.append(
_BasicTrainJob(abs_summary_base_dir=self._summary_base_dir,
entry=info))
with self._lock:
new_cache_items = self._merge_with_disk(basic_train_jobs)
self._cache_items = new_cache_items
for updater in self._updaters.values():
for cache_item in self._cache_items.values():
updater.update_item(cache_item)
def _merge_with_disk(self, disk_train_jobs: Iterable[_BasicTrainJob]):
"""
Merge train jobs in cache with train jobs from disk
This method will remove train jobs not on disk. Call this function with lock for thread safety.
Args:
disk_train_jobs (Iterable[_BasicTrainJob]): Basic train jobs info from disk.
Returns:
dict, a dict containing train jobs to be cached.
"""
new_cache_items = {}
for train_job in disk_train_jobs:
if train_job.train_id not in self._cache_items:
new_cache_items[train_job.train_id] = CachedTrainJob(train_job)
else:
reused_train_job = self._cache_items[train_job.train_id]
reused_train_job.basic_info = train_job
new_cache_items[train_job.train_id] = reused_train_job
return new_cache_items
def register_folder_analyzer(self, analyzer):
"""Register folder analyzer."""
self._summary_watcher.register_folder_analyzer(analyzer)
@property
def cache_items(self):
"""Get cache items."""
return self._cache_items
class DataLoaderGenerator(LoaderGenerator):
"""
DataLoaderGenerator generate a loader_dict of loader from summary logs.
Each loader helps deal the data of the events.
It helps DataManager to generate loaders.
"""
def __init__(self, summary_path):
"""
Init DataLoaderGenerator.
Args:
summary_path (str): A directory path, e.g. '/data/ImageNet/'.
"""
self._summary_path = self._check_and_normalize_summary_path(
summary_path)
self._summary_watcher = SummaryWatcher()
def _check_and_normalize_summary_path(self, summary_path):
"""
Check and normalize summary path.
Args:
summary_path (str): A directory path, e.g. '/data/ImageNet/'.
Returns:
str, normalized summary path.
"""
if summary_path is None:
logger.warning(
"Summary path is None. It will not init data loader generator."
)
raise ParamValueError("Summary path is None.")
summary_path = os.path.realpath(summary_path)
return summary_path
def generate_loaders(self, loader_pool):
"""
Generate loader from summary path, if summary path is empty, will return empty list.
Args:
loader_pool (dict[str, LoaderStruct]): Current loader pool in data_manager.
Returns:
dict[str, LoaderStruct], a dict of `Loader`.
"""
loader_dict = {}
if not FileHandler.exists(self._summary_path):
logger.warning(
"Summary path does not exist. It will not start loading events data. "
"Current path is %r.", self._summary_path)
return loader_dict
dir_map_mtime_dict = {}
min_modify_time = None
summaries_info = self._summary_watcher.list_summary_directories(
self._summary_path)
for item in summaries_info:
relative_path = item.get("relative_path")
current_dir = FileHandler.join(self._summary_path, relative_path)
dataloader = DataLoader(current_dir)
if not dataloader.has_valid_files():
logger.debug(
"Can not find valid train log file in folder %s , "
"will ignore.", relative_path)
continue
modify_time = item.get("update_time").timestamp()
# if loader exists in loader pool and newer time, update its time
loader_id = self._generate_loader_id(relative_path)
loader = loader_pool.get(loader_id)
if loader is not None and loader.latest_update_time > modify_time:
modify_time = loader.latest_update_time
if not min_modify_time:
# The first load, init min modify time
min_modify_time = modify_time
# We need to find `MAX_DATA_LOADER_SIZE` newly modified folders.
if len(dir_map_mtime_dict) < MAX_DATA_LOADER_SIZE:
if modify_time < min_modify_time:
min_modify_time = modify_time
dir_map_mtime_dict.update({relative_path: modify_time})
else:
if modify_time >= min_modify_time:
dir_map_mtime_dict.update({relative_path: modify_time})
sorted_dir_tuple = sorted(dir_map_mtime_dict.items(),
key=lambda d: d[1])[-MAX_DATA_LOADER_SIZE:]
for relative_path, modify_time in sorted_dir_tuple:
loader_id = self._generate_loader_id(relative_path)
loader = self._generate_loader_by_relative_path(relative_path)
loader_dict.update({loader_id: loader})
return loader_dict
def _generate_loader_by_relative_path(self, relative_path):
"""
Generate loader by relative path.
Args:
relative_path (str): Relative path of a summary directory, e.g. './log1'.
Returns:
dict[str, LoaderStruct], a dict of `Loader`.
"""
current_dir = os.path.realpath(
FileHandler.join(self._summary_path, relative_path))
data_loader = DataLoader(current_dir)
loader_id = self._generate_loader_id(relative_path)
loader = LoaderStruct(
loader_id=loader_id,
name=self._generate_loader_name(relative_path),
path=current_dir,
latest_update_time=FileHandler.file_stat(current_dir).mtime,
data_loader=data_loader)
return loader
def _generate_loader_id(self, relative_path):
"""
Generate loader id from relative path.
Args:
relative_path (str): Relative path of a summary directory, e.g. './log1'.
Returns:
str, loader_id for `Loader`.
"""
loader_id = relative_path
return loader_id
def _generate_loader_name(self, relative_path):
"""
Generate loader name from relative path.
Args:
relative_path (str): Relative path of a summary directory, e.g. './log1'.
Returns:
str, loader_name for `Loader`.
"""
loader_name = relative_path
return loader_name
def _get_relative_path_from_train_id(self, train_id):
"""
Get relative from train_id.
Args:
train_id (str): Train ID of a summary directory, e.g. './log1'.
Returns:
str, relative path of `Loader`.
"""
relative_path = train_id
return relative_path
def check_train_job_exist(self, train_id):
"""
Check if train job exists.
Args:
train_id (str): Train ID of a summary directory, e.g. './log1'.
Returns:
bool, if train job exists, return True.
"""
if not self._is_train_id_valid(train_id):
return False
relative_path = self._get_relative_path_from_train_id(train_id)
if self._summary_watcher.is_summary_directory(self._summary_path,
relative_path):
return True
return False
def _is_train_id_valid(self, train_id):
"""
Check if train_id is valid.
Args:
train_id (str): Train ID of a summary directory, e.g. './log1'.
Returns:
bool, if train id is valid, return True.
"""
if not train_id.startswith('./'):
logger.warning("The train_id does not start with './'.")
return False
if len(train_id.split("/")) > 2:
logger.warning("The train_id contains multiple '/'.")
return False
return True
def generate_loader_by_train_id(self, train_id):
"""
Generate loader by train_id.
Args:
train_id (str): Train ID of a summary directory, e.g. './log1'.
Returns:
dict[str, LoaderStruct], a dict of `Loader`.
"""
relative_path = self._get_relative_path_from_train_id(train_id)
loader = self._generate_loader_by_relative_path(relative_path)
return loader