Ejemplo n.º 1
0
    def test_list_tensors_success(self):
        """Test list_tensors method success."""
        summary_base_dir = tempfile.mkdtemp()
        train_job_01 = 'train_01'
        name_01 = 'train_job_01'
        log_path_01 = os.path.join(summary_base_dir, 'dir1')
        self._make_path_and_file_list(log_path_01)
        modify_time_01 = 1575460551.9777446
        loader_01 = DataLoader(log_path_01)

        ms_loader = MSDataLoader(log_path_01)
        event_data = EventsData()
        mock_obj = mock.MagicMock()
        mock_obj.samples.return_value = {'test result'}
        tag = 'image'
        event_data._reservoir_by_tag = {tag: mock_obj}
        ms_loader._events_data = event_data
        loader_01._loader = ms_loader

        loader = LoaderStruct(loader_id=train_job_01,
                              name=name_01,
                              path=log_path_01,
                              latest_update_time=modify_time_01,
                              data_loader=loader_01)
        loader_pool = {train_job_01: loader}
        d_manager = DataManager([DataLoaderGenerator(summary_base_dir)])
        d_manager._status = DataManagerStatus.LOADING.value
        d_manager._loader_pool = loader_pool

        res = d_manager.list_tensors(train_job_01, tag)
        assert res == {'test result'}

        shutil.rmtree(summary_base_dir)
Ejemplo n.º 2
0
    def test_list_tensors_with_keyerror(self):
        """Test list_tensors method with parameter tag raises keyerror."""
        summary_base_dir = tempfile.mkdtemp()
        train_job_01 = 'train_01'
        name_01 = 'train_job_01'
        log_path_01 = os.path.join(summary_base_dir, 'dir1')
        self._make_path_and_file_list(log_path_01)
        modify_time_01 = 1575460551.9777446
        ms_loader = MSDataLoader(log_path_01)
        loader_01 = DataLoader(log_path_01)
        loader_01._loader = ms_loader

        loader = LoaderStruct(loader_id=train_job_01,
                              name=name_01,
                              path=log_path_01,
                              latest_update_time=modify_time_01,
                              data_loader=loader_01)
        loader_pool = {train_job_01: loader}
        d_manager = DataManager([DataLoaderGenerator(summary_base_dir)])
        d_manager._status = DataManagerStatus.LOADING.value
        d_manager._loader_pool = loader_pool
        tag = 'image'
        with pytest.raises(ParamValueError):
            d_manager.list_tensors(train_job_01, tag)

        shutil.rmtree(summary_base_dir)
Ejemplo n.º 3
0
 def _make_loader_dict(self, summary_base_dir, dir_num, start_index=0):
     """Utils function for making loader dict."""
     if os.path.exists(summary_base_dir):
         shutil.rmtree(summary_base_dir)
     os.mkdir(summary_base_dir)
     loader_dict = dict()
     for i in range(dir_num):
         log_dir = os.path.join(summary_base_dir, f'job{start_index+i}')
         os.mkdir(log_dir)
         data_loader = DataLoader(log_dir)
         loader_id = log_dir.replace(summary_base_dir, ".")
         loader = LoaderStruct(loader_id=loader_id,
                               name=loader_id,
                               path=log_dir,
                               latest_update_time=time.time() + i,
                               data_loader=data_loader)
         loader_dict.update({loader_id: loader})
     return loader_dict
Ejemplo n.º 4
0
    def _generate_loader_by_relative_path(self, relative_path):
        """
        Generate loader by relative path.

        Args:
            relative_path (str): Relative path of a summary directory, e.g. './log1'.

        Returns:
            dict[str, LoaderStruct], a dict of `Loader`.
        """
        current_dir = os.path.realpath(FileHandler.join(self._summary_path, relative_path))
        data_loader = DataLoader(current_dir)
        loader_id = self._generate_loader_id(relative_path)
        loader = LoaderStruct(loader_id=loader_id,
                              name=self._generate_loader_name(relative_path),
                              path=current_dir,
                              latest_update_time=FileHandler.file_stat(current_dir).mtime,
                              data_loader=data_loader)
        return loader