def test_list_tensors_success(self): """Test list_tensors method success.""" summary_base_dir = tempfile.mkdtemp() train_job_01 = 'train_01' name_01 = 'train_job_01' log_path_01 = os.path.join(summary_base_dir, 'dir1') self._make_path_and_file_list(log_path_01) modify_time_01 = 1575460551.9777446 loader_01 = DataLoader(log_path_01) ms_loader = MSDataLoader(log_path_01) event_data = EventsData() mock_obj = mock.MagicMock() mock_obj.samples.return_value = {'test result'} tag = 'image' event_data._reservoir_by_tag = {tag: mock_obj} ms_loader._events_data = event_data loader_01._loader = ms_loader loader = LoaderStruct(loader_id=train_job_01, name=name_01, path=log_path_01, latest_update_time=modify_time_01, data_loader=loader_01) loader_pool = {train_job_01: loader} d_manager = DataManager([DataLoaderGenerator(summary_base_dir)]) d_manager._status = DataManagerStatus.LOADING.value d_manager._loader_pool = loader_pool res = d_manager.list_tensors(train_job_01, tag) assert res == {'test result'} shutil.rmtree(summary_base_dir)
def test_list_tensors_with_keyerror(self): """Test list_tensors method with parameter tag raises keyerror.""" summary_base_dir = tempfile.mkdtemp() train_job_01 = 'train_01' name_01 = 'train_job_01' log_path_01 = os.path.join(summary_base_dir, 'dir1') self._make_path_and_file_list(log_path_01) modify_time_01 = 1575460551.9777446 ms_loader = MSDataLoader(log_path_01) loader_01 = DataLoader(log_path_01) loader_01._loader = ms_loader loader = LoaderStruct(loader_id=train_job_01, name=name_01, path=log_path_01, latest_update_time=modify_time_01, data_loader=loader_01) loader_pool = {train_job_01: loader} d_manager = DataManager([DataLoaderGenerator(summary_base_dir)]) d_manager._status = DataManagerStatus.LOADING.value d_manager._loader_pool = loader_pool tag = 'image' with pytest.raises(ParamValueError): d_manager.list_tensors(train_job_01, tag) shutil.rmtree(summary_base_dir)
def test_load_with_not_file_list(self): """Test loading method with empty file list.""" loader = DataLoader(self._summary_dir) with pytest.raises(SummaryLogPathInvalid): loader.load() assert 'No valid files can be loaded' in str( MockLogger.log_msg['warning'])
def test_load_with_invalid_file_list(self): """Test loading method with valid path and invalid file_list.""" file_list = ['summary.abc01', 'summary.abc02'] self._generate_files(self._summary_dir, file_list) loader = DataLoader(self._summary_dir) with pytest.raises(SummaryLogPathInvalid): loader.load() assert 'No valid files can be loaded' in str(MockLogger.log_msg['warning'])
def test_load_success(self): """Test loading method with valid path and file_list.""" dir_path = tempfile.NamedTemporaryFile().name if not os.path.exists(dir_path): os.mkdir(dir_path) file_list = ['summary.001', 'summary.002'] self._generate_files(dir_path, file_list) dataloader = DataLoader(dir_path) dataloader.load() assert dataloader._loader is not None shutil.rmtree(dir_path)
def _make_loader_dict(self, summary_base_dir, dir_num, start_index=0): """Utils function for making loader dict.""" if os.path.exists(summary_base_dir): shutil.rmtree(summary_base_dir) os.mkdir(summary_base_dir) loader_dict = dict() for i in range(dir_num): log_dir = os.path.join(summary_base_dir, f'job{start_index+i}') os.mkdir(log_dir) data_loader = DataLoader(log_dir) loader_id = log_dir.replace(summary_base_dir, ".") loader = LoaderStruct(loader_id=loader_id, name=loader_id, path=log_dir, latest_update_time=time.time() + i, data_loader=data_loader) loader_dict.update({loader_id: loader}) return loader_dict
def _generate_loader_by_relative_path(self, relative_path): """ Generate loader by relative path. Args: relative_path (str): Relative path of a summary directory, e.g. './log1'. Returns: dict[str, LoaderStruct], a dict of `Loader`. """ current_dir = os.path.realpath(FileHandler.join(self._summary_path, relative_path)) data_loader = DataLoader(current_dir) loader_id = self._generate_loader_id(relative_path) loader = LoaderStruct(loader_id=loader_id, name=self._generate_loader_name(relative_path), path=current_dir, latest_update_time=FileHandler.file_stat(current_dir).mtime, data_loader=data_loader) return loader
def generate_loaders(self, loader_pool): """ Generate loader from summary path, if summary path is empty, will return empty list. Args: loader_pool (dict[str, LoaderStruct]): Current loader pool in data_manager. Returns: dict[str, LoaderStruct], a dict of `Loader`. """ loader_dict = {} if not FileHandler.exists(self._summary_path): logger.warning( "Summary path does not exist. It will not start loading events data. " "Current path is %r.", self._summary_path) return loader_dict dir_map_mtime_dict = {} min_modify_time = None summaries_info = self._summary_watcher.list_summary_directories( self._summary_path) for item in summaries_info: relative_path = item.get("relative_path") current_dir = FileHandler.join(self._summary_path, relative_path) dataloader = DataLoader(current_dir) if not dataloader.has_valid_files(): logger.debug( "Can not find valid train log file in folder %s , " "will ignore.", relative_path) continue modify_time = item.get("update_time").timestamp() # if loader exists in loader pool and newer time, update its time loader_id = self._generate_loader_id(relative_path) loader = loader_pool.get(loader_id) if loader is not None and loader.latest_update_time > modify_time: modify_time = loader.latest_update_time if not min_modify_time: # The first load, init min modify time min_modify_time = modify_time # We need to find `MAX_DATA_LOADER_SIZE` newly modified folders. if len(dir_map_mtime_dict) < MAX_DATA_LOADER_SIZE: if modify_time < min_modify_time: min_modify_time = modify_time dir_map_mtime_dict.update({relative_path: modify_time}) else: if modify_time >= min_modify_time: dir_map_mtime_dict.update({relative_path: modify_time}) sorted_dir_tuple = sorted(dir_map_mtime_dict.items(), key=lambda d: d[1])[-MAX_DATA_LOADER_SIZE:] for relative_path, modify_time in sorted_dir_tuple: loader_id = self._generate_loader_id(relative_path) loader = self._generate_loader_by_relative_path(relative_path) loader_dict.update({loader_id: loader}) return loader_dict