class _BriefCacheManager(_BaseCacheManager):
"""A cache manager that holds all disk train jobs on disk."""
def __init__(self, summary_base_dir):
super(_BriefCacheManager, self).__init__(summary_base_dir)
self._summary_watcher = SummaryWatcher()
def cache_train_job(self, train_id):
"""
Cache given train job.
All disk train jobs are cached on every reload, so this method always return false.
Args:
train_id (str): Train Id.
"""
if train_id in self._cache_items:
self._cache_items[train_id].update_access_time()
return False
def update_cache(self, executor):
"""Update cache."""
logger.info('Start to update BriefCacheManager.')
summaries_info = self._summary_watcher.list_summary_directories(
self._summary_base_dir)
basic_train_jobs = []
for info in summaries_info:
basic_train_jobs.append(
_BasicTrainJob(abs_summary_base_dir=self._summary_base_dir,
entry=info))
with self._lock:
new_cache_items = self._merge_with_disk(basic_train_jobs)
self._cache_items = new_cache_items
for updater in self._updaters.values():
for cache_item in self._cache_items.values():
updater.update_item(cache_item)
def _merge_with_disk(self, disk_train_jobs: Iterable[_BasicTrainJob]):
"""
Merge train jobs in cache with train jobs from disk
This method will remove train jobs not on disk. Call this function with lock for thread safety.
Args:
disk_train_jobs (Iterable[_BasicTrainJob]): Basic train jobs info from disk.
Returns:
dict, a dict containing train jobs to be cached.
"""
new_cache_items = {}
for train_job in disk_train_jobs:
if train_job.train_id not in self._cache_items:
new_cache_items[train_job.train_id] = CachedTrainJob(train_job)
else:
reused_train_job = self._cache_items[train_job.train_id]
reused_train_job.basic_info = train_job
new_cache_items[train_job.train_id] = reused_train_job
return new_cache_items
def register_folder_analyzer(self, analyzer):
"""Register folder analyzer."""
self._summary_watcher.register_folder_analyzer(analyzer)
@property
def cache_items(self):
"""Get cache items."""
return self._cache_items
class DataLoaderGenerator(LoaderGenerator):
"""
DataLoaderGenerator generate a loader_dict of loader from summary logs.
Each loader helps deal the data of the events.
It helps DataManager to generate loaders.
"""
def __init__(self, summary_path):
"""
Init DataLoaderGenerator.
Args:
summary_path (str): A directory path, e.g. '/data/ImageNet/'.
"""
self._summary_path = self._check_and_normalize_summary_path(summary_path)
self._summary_watcher = SummaryWatcher()
def register_folder_analyzer(self, analyzer):
"""Register folder analyzer."""
self._summary_watcher.register_folder_analyzer(analyzer)
def _check_and_normalize_summary_path(self, summary_path):
"""
Check and normalize summary path.
Args:
summary_path (str): A directory path, e.g. '/data/ImageNet/'.
Returns:
str, normalized summary path.
"""
if summary_path is None:
logger.warning("Summary path is None. It will not init data loader generator.")
raise ParamValueError("Summary path is None.")
summary_path = os.path.realpath(summary_path)
return summary_path
def generate_loaders(self, loader_pool):
"""
Generate loader from summary path, if summary path is empty, will return empty list.
Args:
loader_pool (dict[str, LoaderStruct]): Current loader pool in data_manager.
Returns:
dict[str, LoaderStruct], a dict of `Loader`.
"""
loader_dict = {}
if not FileHandler.exists(self._summary_path):
logger.warning("Summary path does not exist. It will not start loading events data. "
"Current path is %r.", self._summary_path)
return loader_dict
dir_map_mtime_dict = {}
min_modify_time = None
summaries_info = self._summary_watcher.list_summary_directories(self._summary_path)
for item in summaries_info:
relative_path = item.get("relative_path")
current_dir = FileHandler.join(self._summary_path, relative_path)
dataloader = DataLoader(current_dir)
if not dataloader.has_valid_files():
logger.debug("Can not find valid train log file in folder %s , "
"will ignore.", relative_path)
continue
modify_time = item.get("update_time").timestamp()
# if loader exists in loader pool and newer time, update its time
loader_id = self._generate_loader_id(relative_path)
loader = loader_pool.get(loader_id)
if loader is not None and loader.latest_update_time > modify_time:
modify_time = loader.latest_update_time
if not min_modify_time:
# The first load, init min modify time
min_modify_time = modify_time
# We need to find `MAX_DATA_LOADER_SIZE` newly modified folders.
if len(dir_map_mtime_dict) < MAX_DATA_LOADER_SIZE:
if modify_time < min_modify_time:
min_modify_time = modify_time
dir_map_mtime_dict.update({relative_path: modify_time})
else:
if modify_time >= min_modify_time:
dir_map_mtime_dict.update({relative_path: modify_time})
sorted_dir_tuple = sorted(dir_map_mtime_dict.items(),
key=lambda d: d[1])[-MAX_DATA_LOADER_SIZE:]
for relative_path, modify_time in sorted_dir_tuple:
loader_id = self._generate_loader_id(relative_path)
loader = self._generate_loader_by_relative_path(relative_path)
loader_dict.update({loader_id: loader})
return loader_dict
def _generate_loader_by_relative_path(self, relative_path):
"""
Generate loader by relative path.
Args:
relative_path (str): Relative path of a summary directory, e.g. './log1'.
Returns:
dict[str, LoaderStruct], a dict of `Loader`.
"""
current_dir = os.path.realpath(FileHandler.join(self._summary_path, relative_path))
data_loader = DataLoader(current_dir)
loader_id = self._generate_loader_id(relative_path)
loader = LoaderStruct(loader_id=loader_id,
name=self._generate_loader_name(relative_path),
path=current_dir,
latest_update_time=FileHandler.file_stat(current_dir).mtime,
data_loader=data_loader)
return loader
def _generate_loader_id(self, relative_path):
"""
Generate loader id from relative path.
Args:
relative_path (str): Relative path of a summary directory, e.g. './log1'.
Returns:
str, loader_id for `Loader`.
"""
loader_id = relative_path
return loader_id
def _generate_loader_name(self, relative_path):
"""
Generate loader name from relative path.
Args:
relative_path (str): Relative path of a summary directory, e.g. './log1'.
Returns:
str, loader_name for `Loader`.
"""
loader_name = relative_path
return loader_name
def _get_relative_path_from_train_id(self, train_id):
"""
Get relative from train_id.
Args:
train_id (str): Train ID of a summary directory, e.g. './log1'.
Returns:
str, relative path of `Loader`.
"""
relative_path = train_id
return relative_path
def check_train_job_exist(self, train_id):
"""
Check if train job exists.
Args:
train_id (str): Train ID of a summary directory, e.g. './log1'.
Returns:
bool, if train job exists, return True.
"""
if not self._is_train_id_valid(train_id):
return False
relative_path = self._get_relative_path_from_train_id(train_id)
if self._summary_watcher.is_summary_directory(self._summary_path, relative_path):
return True
return False
def _is_train_id_valid(self, train_id):
"""
Check if train_id is valid.
Args:
train_id (str): Train ID of a summary directory, e.g. './log1'.
Returns:
bool, if train id is valid, return True.
"""
if not train_id.startswith('./'):
logger.warning("The train_id does not start with './'.")
return False
if len(train_id.split("/")) > 2:
logger.warning("The train_id contains multiple '/'.")
return False
return True
def generate_loader_by_train_id(self, train_id):
"""
Generate loader by train_id.
Args:
train_id (str): Train ID of a summary directory, e.g. './log1'.
Returns:
dict[str, LoaderStruct], a dict of `Loader`.
"""
relative_path = self._get_relative_path_from_train_id(train_id)
try:
loader = self._generate_loader_by_relative_path(relative_path)
except PathNotExistError as ex:
raise TrainJobNotExistError(str(ex))
return loader