def test_softmax_model(fake_dataset: DictsDataset): tf.logging.set_verbosity(tf.logging.DEBUG) features = fake_dataset.features steps = np.random.randint(2, 5) run_data = gen.run_data(model=MnistSoftmaxModel()) estimator = training.create_estimator(run_data) estimator.train(input_fn=lambda: fake_dataset.as_tuple(), steps=steps) names = [gen.random_str() for _ in range(2)] eval_results = estimator.evaluate(input_fn=lambda: fake_dataset.as_tuple(), steps=1, name=names[0]) estimator.evaluate(input_fn=lambda: fake_dataset.as_tuple(), steps=1, name=names[1]) _assert_log_dirs(filenames.get_run_logs_data_dir(run_data), names) loss = eval_results['loss'] global_step = eval_results['global_step'] accuracy = eval_results['accuracy'] assert loss.shape == () assert global_step == steps assert accuracy.shape == () predictions_generator = estimator.predict(input_fn=lambda: features) for _ in range((list(features.values())[0]).shape[0]): predictions = next(predictions_generator) assert predictions['probabilities'].shape == (2, ) assert predictions[consts.INFERENCE_CLASSES].shape == ()
def test_should_train_with_each_model(injected_raw_data_provider): tf.logging.set_verbosity(tf.logging.DEBUG) run_data = gen.run_data(model=injected_raw_data_provider()) before_run.prepare_env([], run_data) training.train(run_data) verify_log_directory(run_data, config[consts.EXCLUDED_KEYS])
def test_should_create_correct_infer_directory_for_single_model_launcher( patched_params): run_data = gen.run_data() result = filenames.get_infer_dir(run_data) expected = filenames._get_home_infer_dir() / run_data.model.summary assert result == expected
def test_should_not_override_old_inference_log(patched_dataset_reading, patched_params): run_data = gen.run_data(model=MnistContrastiveModel()) inference.single_run_inference(run_data=run_data, show=False) infer_results_dir_path = filenames.get_infer_dir(run_data) old_cr_times = [ x.stat().st_ctime for x in list(Path(infer_results_dir_path).iterdir()) if x.suffix == consts.LOG ] inference.single_run_inference(run_data=run_data, show=False) new_cr_times = [ x.stat().st_ctime for x in list(Path(infer_results_dir_path).iterdir()) if x.suffix == consts.LOG ] assert len(old_cr_times) == 1 assert len(new_cr_times) == 2 assert old_cr_times[0] in new_cr_times
def test_should_not_delete_existing_launcher_directory_for_experiment_launcher_when_its_not_first_run( when, rmtree_mock): run_data = gen.run_data(is_experiment=True, run_no=2) result = before_run._prepare_launcher_dir(run_data) assert result is None rmtree_mock.assert_not_called()
def test_should_not_delete_existing_launcher_directory_for_default_launcher( when, rmtree_mock): run_data = gen.run_data(is_experiment=False, run_no=1) result = before_run._prepare_launcher_dir(run_data) assert result is None rmtree_mock.assert_not_called()
def test_should_create_correct_infer_directory_for_experiment_launcher( patched_params): run_data = gen.run_data(is_experiment=True) result = filenames.get_infer_dir(run_data) expected = filenames._get_home_infer_dir( ) / run_data.launcher_name / run_data.model.summary assert result == expected
def test_should_not_throw_if_model_has_more_checkpoints(): run_data = gen.run_data(with_model_dir=True) model_dir = filenames.get_run_logs_data_dir(run_data) empty_checkpoints = [(model_dir / ("model.ckpt-{}.foobar".format(x))) for x in range(2)] [f.write_text("this is sparta") for f in empty_checkpoints] inference.single_run_inference(run_data=run_data, show=False)
def test_should_provide_single_run_data_for_experiment_launcher(mocker): launcher = FakeExperimentLauncher([FakeModel(), FakeModel()]) run_data = gen.run_data() mocker.patch('src.estimator.launcher.providing_launcher.provide_launcher', return_value=launcher) mocker.patch('src.utils.utils.user_run_selection', return_value=run_data) result = providing_launcher.provide_single_run_data() assert result == run_data
def test_should_throw_if_model_has_only_0_step_checkpoints(): run_data = gen.run_data(with_model_dir=True) model_dir = filenames.get_run_logs_data_dir(run_data) empty_checkpoints = [ model_dir / ("model.ckpt-0.foobar{}".format(x)) for x in range(5) ] [f.write_text("this is sparta") for f in empty_checkpoints] with pytest.raises(AssertionError): inference.single_run_inference(run_data=run_data, show=False)
def test_should_create_summaries_for_different_models(patched_dataset_reading, patched_params): model = patched_dataset_reading.param run_data = gen.run_data(model=model()) inference.single_run_inference(run_data=run_data, show=False) infer_results_dir_path = filenames.get_infer_dir(run_data) assert utils.check_filepath(infer_results_dir_path, is_directory=True, is_empty=False)
def test_should_delete_existing_run_directory_for_default_launcher( when, rmtree_mock): run_data = gen.run_data(is_experiment=False) run_logs_dir = filenames.get_run_logs_data_dir(run_data) when(utils).check_filepath(filename=run_logs_dir, exists=True, is_directory=True, is_empty=ANY).thenReturn(True) before_run._prepare_dirs(None, run_data) rmtree_mock.assert_called_once_with(str(run_logs_dir))
def test_create_pair_summaries(patched_dataset_reading): provider = patched_dataset_reading.param run_data = gen.run_data(model=FakeModel(data_provider=provider())) dir_with_pair_summaries = filenames.get_run_logs_data_dir( run_data) / 'features' assert utils.check_filepath(dir_with_pair_summaries, exists=False) image_summaries.create_pair_summaries(run_data) assert utils.check_filepath(dir_with_pair_summaries, is_directory=True, is_empty=False) assert len(list(dir_with_pair_summaries.iterdir())) == 1
def test_should_delete_existing_launcher_directory_for_experiment_launcher_during_first_run( when, rmtree_mock): run_data = gen.run_data(is_experiment=True, run_no=1) launcher_dir = filenames.get_launcher_dir(run_data) when(utils).check_filepath(filename=launcher_dir, exists=True, is_directory=True, is_empty=ANY).thenReturn(True) result = before_run._prepare_launcher_dir(run_data) assert result == launcher_dir rmtree_mock.assert_called_once_with(str(launcher_dir))
def test_should_create_correct_number_of_inference_files( patched_dataset_reading, patched_params): model = patched_dataset_reading.param() run_data = gen.run_data(model=model) inference.single_run_inference(run_data=run_data, show=False) infer_results_dir_path = filenames.get_infer_dir(run_data) expected_file_count = 4 if model.produces_2d_embedding else 2 assert utils.check_filepath(infer_results_dir_path, is_directory=True, is_empty=False, expected_len=expected_file_count)
def test_should_call_in_memory_evaluator_hooks(input_fn_spies, patched_dataset_reading, patched_excluded): (train_input_fn_spy, eval_input_fn_spy, eval_with_excludes_fn_spy) = input_fn_spies run_data = gen.run_data(model=patched_dataset_reading.param()) before_run.prepare_env([], run_data) training.train(run_data) train_input_fn_spy.assert_called_once() eval_input_fn_spy.assert_called_once() assert eval_with_excludes_fn_spy.call_count == ( 1 if config[consts.EXCLUDED_KEYS] else 0) verify_log_directory(run_data, config[consts.EXCLUDED_KEYS]) assert run_data.model.model_fn_calls == (3 if config[consts.EXCLUDED_KEYS] else 2)
def test_should_override_plots_with_newer_inference(patched_dataset_reading, patched_params): run_data = gen.run_data(model=MnistContrastiveModel()) inference.single_run_inference(run_data=run_data, show=False) infer_results_dir_path = filenames.get_infer_dir(run_data) old_cr_times = [ x.stat().st_ctime for x in list(Path(infer_results_dir_path).iterdir()) if x.suffix != consts.LOG ] inference.single_run_inference(run_data=run_data, show=False) new_cr_times = [ x.stat().st_ctime for x in list(Path(infer_results_dir_path).iterdir()) if x.suffix != consts.LOG ] var = [(x not in new_cr_times) for x in old_cr_times] assert_that(var, only_contains(True))
def test_run_text_logs_dir(run_dir_mock): run_logging_dir = filenames.get_run_text_logs_dir(gen.run_data()) assert_that(str(run_logging_dir), ends_with('baz/text_logs'))
def test_should_pass_model_dir_to_estimator(): model = FakeModel() run_data = gen.run_data(model) estimator = training.create_estimator(run_data) model_dir = estimator.params[consts.MODEL_DIR] assert model_dir == str(filenames.get_run_logs_data_dir(run_data))
def test_should_throw_if_model_has_no_checkpoints(): run_data = gen.run_data(with_model_dir=True) with pytest.raises(AssertionError): inference.single_run_inference(run_data=run_data, show=False)
def test_should_throw_if_model_dir_not_exists(): run_data = gen.run_data(with_model_dir=False) with pytest.raises(AssertionError): inference.single_run_inference(run_data=run_data, show=False)