def test_list_tensors_success(self):
"""Test list_tensors method success."""
summary_base_dir = tempfile.mkdtemp()
train_job_01 = 'train_01'
name_01 = 'train_job_01'
log_path_01 = os.path.join(summary_base_dir, 'dir1')
self._make_path_and_file_list(log_path_01)
modify_time_01 = 1575460551.9777446
loader_01 = DataLoader(log_path_01)
ms_loader = MSDataLoader(log_path_01)
event_data = EventsData()
mock_obj = mock.MagicMock()
mock_obj.samples.return_value = {'test result'}
tag = 'image'
event_data._reservoir_by_tag = {tag: mock_obj}
ms_loader._events_data = event_data
loader_01._loader = ms_loader
loader = LoaderStruct(loader_id=train_job_01,
name=name_01,
path=log_path_01,
latest_update_time=modify_time_01,
data_loader=loader_01)
loader_pool = {train_job_01: loader}
d_manager = DataManager([DataLoaderGenerator(summary_base_dir)])
d_manager._status = DataManagerStatus.LOADING.value
d_manager._loader_pool = loader_pool
res = d_manager.list_tensors(train_job_01, tag)
assert res == {'test result'}
shutil.rmtree(summary_base_dir)
def test_list_tensors_with_keyerror(self):
"""Test list_tensors method with parameter tag raises keyerror."""
summary_base_dir = tempfile.mkdtemp()
train_job_01 = 'train_01'
name_01 = 'train_job_01'
log_path_01 = os.path.join(summary_base_dir, 'dir1')
self._make_path_and_file_list(log_path_01)
modify_time_01 = 1575460551.9777446
ms_loader = MSDataLoader(log_path_01)
loader_01 = DataLoader(log_path_01)
loader_01._loader = ms_loader
loader = LoaderStruct(loader_id=train_job_01,
name=name_01,
path=log_path_01,
latest_update_time=modify_time_01,
data_loader=loader_01)
loader_pool = {train_job_01: loader}
d_manager = DataManager([DataLoaderGenerator(summary_base_dir)])
d_manager._status = DataManagerStatus.LOADING.value
d_manager._loader_pool = loader_pool
tag = 'image'
with pytest.raises(ParamValueError):
d_manager.list_tensors(train_job_01, tag)
shutil.rmtree(summary_base_dir)
def test_load_with_not_file_list(self):
"""Test loading method with empty file list."""
loader = DataLoader(self._summary_dir)
with pytest.raises(SummaryLogPathInvalid):
loader.load()
assert 'No valid files can be loaded' in str(
MockLogger.log_msg['warning'])
def test_load_with_invalid_file_list(self):
"""Test loading method with valid path and invalid file_list."""
file_list = ['summary.abc01', 'summary.abc02']
self._generate_files(self._summary_dir, file_list)
loader = DataLoader(self._summary_dir)
with pytest.raises(SummaryLogPathInvalid):
loader.load()
assert 'No valid files can be loaded' in str(MockLogger.log_msg['warning'])
def test_load_success(self):
"""Test loading method with valid path and file_list."""
dir_path = tempfile.NamedTemporaryFile().name
if not os.path.exists(dir_path):
os.mkdir(dir_path)
file_list = ['summary.001', 'summary.002']
self._generate_files(dir_path, file_list)
dataloader = DataLoader(dir_path)
dataloader.load()
assert dataloader._loader is not None
shutil.rmtree(dir_path)
def _make_loader_dict(self, summary_base_dir, dir_num, start_index=0):
"""Utils function for making loader dict."""
if os.path.exists(summary_base_dir):
shutil.rmtree(summary_base_dir)
os.mkdir(summary_base_dir)
loader_dict = dict()
for i in range(dir_num):
log_dir = os.path.join(summary_base_dir, f'job{start_index+i}')
os.mkdir(log_dir)
data_loader = DataLoader(log_dir)
loader_id = log_dir.replace(summary_base_dir, ".")
loader = LoaderStruct(loader_id=loader_id,
name=loader_id,
path=log_dir,
latest_update_time=time.time() + i,
data_loader=data_loader)
loader_dict.update({loader_id: loader})
return loader_dict
def _generate_loader_by_relative_path(self, relative_path):
"""
Generate loader by relative path.
Args:
relative_path (str): Relative path of a summary directory, e.g. './log1'.
Returns:
dict[str, LoaderStruct], a dict of `Loader`.
"""
current_dir = os.path.realpath(FileHandler.join(self._summary_path, relative_path))
data_loader = DataLoader(current_dir)
loader_id = self._generate_loader_id(relative_path)
loader = LoaderStruct(loader_id=loader_id,
name=self._generate_loader_name(relative_path),
path=current_dir,
latest_update_time=FileHandler.file_stat(current_dir).mtime,
data_loader=data_loader)
return loader
def generate_loaders(self, loader_pool):
"""
Generate loader from summary path, if summary path is empty, will return empty list.
Args:
loader_pool (dict[str, LoaderStruct]): Current loader pool in data_manager.
Returns:
dict[str, LoaderStruct], a dict of `Loader`.
"""
loader_dict = {}
if not FileHandler.exists(self._summary_path):
logger.warning(
"Summary path does not exist. It will not start loading events data. "
"Current path is %r.", self._summary_path)
return loader_dict
dir_map_mtime_dict = {}
min_modify_time = None
summaries_info = self._summary_watcher.list_summary_directories(
self._summary_path)
for item in summaries_info:
relative_path = item.get("relative_path")
current_dir = FileHandler.join(self._summary_path, relative_path)
dataloader = DataLoader(current_dir)
if not dataloader.has_valid_files():
logger.debug(
"Can not find valid train log file in folder %s , "
"will ignore.", relative_path)
continue
modify_time = item.get("update_time").timestamp()
# if loader exists in loader pool and newer time, update its time
loader_id = self._generate_loader_id(relative_path)
loader = loader_pool.get(loader_id)
if loader is not None and loader.latest_update_time > modify_time:
modify_time = loader.latest_update_time
if not min_modify_time:
# The first load, init min modify time
min_modify_time = modify_time
# We need to find `MAX_DATA_LOADER_SIZE` newly modified folders.
if len(dir_map_mtime_dict) < MAX_DATA_LOADER_SIZE:
if modify_time < min_modify_time:
min_modify_time = modify_time
dir_map_mtime_dict.update({relative_path: modify_time})
else:
if modify_time >= min_modify_time:
dir_map_mtime_dict.update({relative_path: modify_time})
sorted_dir_tuple = sorted(dir_map_mtime_dict.items(),
key=lambda d: d[1])[-MAX_DATA_LOADER_SIZE:]
for relative_path, modify_time in sorted_dir_tuple:
loader_id = self._generate_loader_id(relative_path)
loader = self._generate_loader_by_relative_path(relative_path)
loader_dict.update({loader_id: loader})
return loader_dict
