def test_caching(self, mock_generate_loaders): """Test caching.""" # Load summaries the first time. job_num = 10 summary_base_dir = tempfile.NamedTemporaryFile().name loader_dict = self._make_loader_dict(summary_base_dir, job_num) expected_loader_ids = list(loader_dict.keys()) mock_generate_loaders.return_value = loader_dict generators = [data_manager.DataLoaderGenerator(summary_base_dir)] mock_data_manager = data_manager.DataManager(generators) mock_data_manager._execute_load_data = Mock() mock_data_manager.start_load_data(reload_interval=0) check_loading_done(mock_data_manager, 3) current_loader_ids = mock_data_manager._loader_pool.keys() assert sorted(current_loader_ids) == sorted(expected_loader_ids) # Add new summaries. new_loader_dict = self._make_loader_dict(summary_base_dir, 6, job_num) loader_dict.update(new_loader_dict) expected_loader_ids.extend(list(loader_dict.keys())) expected_loader_ids = expected_loader_ids[-MAX_DATA_LOADER_SIZE:] # Make sure to finish loading, make it init. mock_data_manager._status = DataManagerStatus.INIT mock_generate_loaders.return_value = loader_dict mock_data_manager.start_load_data(reload_interval=0) check_loading_done(mock_data_manager) current_loader_ids = mock_data_manager._loader_pool.keys() assert sorted(current_loader_ids) == sorted(expected_loader_ids) shutil.rmtree(summary_base_dir)
def load_data(self): """Load data.""" log_operation = LogOperations() self._plugins_id_map = {'image': [], 'scalar': [], 'graph': [], 'histogram': [], 'tensor': []} self._events_names = [] self._train_id_list = [] self._root_dir = tempfile.mkdtemp() for i in range(self._dir_num): dir_path = tempfile.mkdtemp(dir=self._root_dir) tmp_tag_name = self._tag_name + '_' + str(i) event_name = str(i) + "_name" train_id = dir_path.replace(self._root_dir, ".") # Pass timestamp to write to the same file. log_settings = dict(steps=self._steps_list, tag=tmp_tag_name, time=time.time()) if i % 3 != 0: log_operation.generate_log(PluginNameEnum.IMAGE.value, dir_path, log_settings) self._plugins_id_map['image'].append(train_id) if i % 3 != 1: log_operation.generate_log(PluginNameEnum.SCALAR.value, dir_path, log_settings) self._plugins_id_map['scalar'].append(train_id) if i % 3 != 2: log_operation.generate_log(PluginNameEnum.GRAPH.value, dir_path, log_settings) self._plugins_id_map['graph'].append(train_id) self._events_names.append(event_name) self._train_id_list.append(train_id) self._generated_path.append(self._root_dir) self._mock_data_manager = data_manager.DataManager(self._root_dir) self._mock_data_manager.start_load_data().join()
def test_caching(self, mock_generate_loaders): """Test caching.""" # Load summaries the first time. job_num = 10 summary_base_dir = tempfile.NamedTemporaryFile().name loader_dict = self._make_loader_dict(summary_base_dir, job_num) expected_loader_ids = list(loader_dict.keys()) mock_generate_loaders.return_value = loader_dict mock_data_manager = data_manager.DataManager(summary_base_dir) mock_data_manager._detail_cache._execute_loader = Mock() mock_data_manager.start_load_data().join() current_loader_ids = mock_data_manager._detail_cache._loader_pool.keys() assert sorted(current_loader_ids) == sorted(expected_loader_ids) # Add new summaries. new_loader_dict = self._make_loader_dict(summary_base_dir, 6, job_num) loader_dict.update(new_loader_dict) expected_loader_ids.extend(list(loader_dict.keys())) expected_loader_ids = expected_loader_ids[-MAX_DATA_LOADER_SIZE:] mock_generate_loaders.return_value = loader_dict mock_data_manager.start_load_data().join() current_loader_ids = mock_data_manager._detail_cache._loader_pool.keys() assert sorted(current_loader_ids) == sorted(expected_loader_ids) shutil.rmtree(summary_base_dir)
def _init_data_manager(self, steps_list): """ Generate log and init data_manager. Args: steps_list (list): Init steps. """ summary_base_dir = tempfile.mkdtemp() log_dir = tempfile.mkdtemp(dir=summary_base_dir) self._train_id = log_dir.replace(summary_base_dir, ".") self._temp_path, self._images_metadata, self._images_values = LogOperations.generate_log( PluginNameEnum.IMAGE.value, log_dir, dict(steps=steps_list, tag=self._tag_name)) self._generated_path.append(summary_base_dir) self._mock_data_manager = data_manager.DataManager( [DataLoaderGenerator(summary_base_dir)]) self._mock_data_manager.start_load_data(reload_interval=0) # wait for loading done check_loading_done(self._mock_data_manager, time_limit=5)
def test_get_flattened_lineage(self): """Test the function of get_flattened_lineage""" datamanager = data_manager.DataManager(SUMMARY_DIR) datamanager.register_brief_cache_item_updater( LineageCacheItemUpdater()) datamanager.start_load_data().join() data = get_flattened_lineage(datamanager) assert data.get('[U]info') == ['info1']
def load_graph_record(self): """Load graph record.""" summary_base_dir = tempfile.mkdtemp() log_dir = tempfile.mkdtemp(dir=summary_base_dir) self._train_id = log_dir.replace(summary_base_dir, ".") log_operation = LogOperations() self._temp_path, self._graph_dict, _ = log_operation.generate_log(PluginNameEnum.GRAPH.value, log_dir) self._generated_path.append(summary_base_dir) self._mock_data_manager = data_manager.DataManager(summary_base_dir) self._mock_data_manager.start_load_data().join()
def load_tensor_record(self): """Load tensor record.""" summary_base_dir = tempfile.mkdtemp() log_dir = tempfile.mkdtemp(dir=summary_base_dir) self._train_id = log_dir.replace(summary_base_dir, ".") log_operation = LogOperations() self._temp_path, self._tensors, _ = log_operation.generate_log( PluginNameEnum.TENSOR.value, log_dir, dict(step=self._steps_list, tag=self._tag_name)) self._generated_path.append(summary_base_dir) self._mock_data_manager = data_manager.DataManager(summary_base_dir) self._mock_data_manager.start_load_data().join()
def load_no_graph_record(self): """Load no graph record.""" summary_base_dir = tempfile.mkdtemp() log_dir = tempfile.mkdtemp(dir=summary_base_dir) self._train_id = log_dir.replace(summary_base_dir, ".") log_operation = LogOperations() self._temp_path, _, _ = log_operation.generate_log(PluginNameEnum.IMAGE.value, log_dir, dict(steps=self._steps_list, tag="image")) self._generated_path.append(summary_base_dir) self._mock_data_manager = data_manager.DataManager(summary_base_dir) self._mock_data_manager.start_load_data().join()
def load_scalar_record(self): """Load scalar record.""" summary_base_dir = tempfile.mkdtemp() log_dir = tempfile.mkdtemp(dir=summary_base_dir) self._train_id = log_dir.replace(summary_base_dir, ".") log_operation = LogOperations() self._temp_path, self._scalars_metadata, self._scalars_values = log_operation.generate_log( PluginNameEnum.SCALAR.value, log_dir, dict(step=self._steps_list, tag=self._tag_name)) self._generated_path.append(summary_base_dir) self._mock_data_manager = data_manager.DataManager(summary_base_dir) self._mock_data_manager.start_load_data(reload_interval=0) # wait for loading done check_loading_done(self._mock_data_manager, time_limit=5)
def test_start_load_data_success(self): """Test start_load_data method success.""" summary_base_dir = tempfile.mkdtemp() dir_num = 3 train_ids = [] for i in range(dir_num): log_path = os.path.join(summary_base_dir, f'dir{i}') self._make_path_and_file_list(log_path) train_ids.append(f'./dir{i}') data_manager.logger = MockLogger mock_manager = data_manager.DataManager(summary_base_dir) mock_manager.start_load_data().join() assert MockLogger.log_msg['info'] == "Load brief data end, and loader pool size is '3'." shutil.rmtree(summary_base_dir)
def load_graph_record(self): """Load graph record.""" summary_base_dir = tempfile.mkdtemp() log_dir = tempfile.mkdtemp(dir=summary_base_dir) self._train_id = log_dir.replace(summary_base_dir, ".") log_operation = LogOperations() self._temp_path, self._graph_dict, _ = log_operation.generate_log( PluginNameEnum.GRAPH.value, log_dir) self._generated_path.append(summary_base_dir) self._mock_data_manager = data_manager.DataManager(summary_base_dir) self._mock_data_manager.start_load_data(reload_interval=0) # wait for loading done check_loading_done(self._mock_data_manager, time_limit=5)
def load_data(self): """Load data.""" self._plugins_id_map = {'image': [], 'scalar': [], 'graph': []} self._events_names = [] self._train_id_list = [] graph_base_path = os.path.join(os.path.dirname(__file__), os.pardir, "utils", "log_generators", "graph_base.json") self._root_dir = tempfile.mkdtemp() for i in range(self._dir_num): dir_path = tempfile.mkdtemp(dir=self._root_dir) tmp_tag_name = self._tag_name + '_' + str(i) event_name = str(i) + "_name" train_id = dir_path.replace(self._root_dir, ".") # Pass timestamp to write to the same file. log_settings = dict(steps=self._steps_list, tag=tmp_tag_name, graph_base_path=graph_base_path, time=time.time()) if i % 3 != 0: LogOperations.generate_log(PluginNameEnum.IMAGE.value, dir_path, log_settings) self._plugins_id_map['image'].append(train_id) if i % 3 != 1: LogOperations.generate_log(PluginNameEnum.SCALAR.value, dir_path, log_settings) self._plugins_id_map['scalar'].append(train_id) if i % 3 != 2: LogOperations.generate_log(PluginNameEnum.GRAPH.value, dir_path, log_settings) self._plugins_id_map['graph'].append(train_id) self._events_names.append(event_name) self._train_id_list.append(train_id) self._generated_path.append(self._root_dir) self._mock_data_manager = data_manager.DataManager( [DataLoaderGenerator(self._root_dir)]) self._mock_data_manager.start_load_data(reload_interval=0) check_loading_done(self._mock_data_manager, time_limit=30)
def load_no_graph_record(self): """Load no graph record.""" summary_base_dir = tempfile.mkdtemp() log_dir = tempfile.mkdtemp(dir=summary_base_dir) self._train_id = log_dir.replace(summary_base_dir, ".") self._temp_path, _, _ = LogOperations.generate_log( PluginNameEnum.IMAGE.value, log_dir, dict(steps=self._steps_list, tag="image")) self._generated_path.append(summary_base_dir) self._mock_data_manager = data_manager.DataManager( [DataLoaderGenerator(summary_base_dir)]) self._mock_data_manager.start_load_data(reload_interval=0) # wait for loading done check_loading_done(self._mock_data_manager, time_limit=5)
def test_start_load_data_success(self): """Test start_load_data method success.""" summary_base_dir = tempfile.mkdtemp() train_ids = [] for i in range(3): log_path = os.path.join(summary_base_dir, f'dir{i}') self._make_path_and_file_list(log_path) train_ids.append(f'./dir{i}') data_manager.logger = MockLogger mock_manager = data_manager.DataManager(summary_base_dir) mock_manager.start_load_data(reload_interval=0) check_loading_done(mock_manager) assert MockLogger.log_msg['info'] == "Load event data end, status: 'DONE', " \ "and loader pool size is '3'." shutil.rmtree(summary_base_dir)
def _init_data_manager(self, steps_list): """ Generate log and init data_manager. Args: steps_list (list): Init steps. """ summary_base_dir = tempfile.mkdtemp() log_dir = tempfile.mkdtemp(dir=summary_base_dir) self._train_id = log_dir.replace(summary_base_dir, ".") log_operation = LogOperations() self._temp_path, self._images_metadata, self._images_values = log_operation.generate_log( PluginNameEnum.IMAGE.value, log_dir, dict(steps=steps_list, tag=self._tag_name)) self._generated_path.append(summary_base_dir) self._mock_data_manager = data_manager.DataManager(summary_base_dir) self._mock_data_manager.start_load_data().join()
def load_graph_record(self): """Load graph record.""" summary_base_dir = tempfile.mkdtemp() log_dir = tempfile.mkdtemp(dir=summary_base_dir) self._train_id = log_dir.replace(summary_base_dir, ".") graph_base_path = os.path.join(os.path.dirname(__file__), os.pardir, "utils", "log_generators", "graph_base.json") self._temp_path, self._graph_dict = LogOperations.generate_log( PluginNameEnum.GRAPH.value, log_dir, dict(graph_base_path=graph_base_path)) self._generated_path.append(summary_base_dir) self._mock_data_manager = data_manager.DataManager( [DataLoaderGenerator(summary_base_dir)]) self._mock_data_manager.start_load_data(reload_interval=0) # wait for loading done check_loading_done(self._mock_data_manager, time_limit=5)