def test_should_return_user_run_selection(mocker):
from estimator.training.integration.test_integration_training import FakeExperimentLauncher
model = FakeModel()
launcher = FakeExperimentLauncher(
[FakeModel(), FakeModel(), model,
FakeModel()])
mocker.patch('builtins.input', return_value='2')
assert utils.user_run_selection(launcher).model == model
def test_should_provide_single_run_data_for_experiment_launcher(mocker):
launcher = FakeExperimentLauncher([FakeModel(), FakeModel()])
run_data = gen.run_data()
mocker.patch('src.estimator.launcher.providing_launcher.provide_launcher',
return_value=launcher)
mocker.patch('src.utils.utils.user_run_selection', return_value=run_data)
result = providing_launcher.provide_single_run_data()
assert result == run_data
def test_should_train_with_each_model(mocker):
cnn_model_uno = FakeModel()
cnn_model_dos = FakeModel()
launcher = FakeExperimentLauncher([cnn_model_uno, cnn_model_dos])
before_run_mock, in_memory_training_mock = prepare_mocks(launcher, mocker)
import sys
training.main(sys.argv)
assert before_run_mock.call_count == 2
assert in_memory_training_mock.call_count == 2
model_arguments = get_model_called_with(before_run_mock)
assert_that(model_arguments, contains(*[x for x in launcher.runs_data]))
def test_should_log_when_flag_not_defined_during_training(mocker):
launcher = DefaultLauncher([(FakeModel())])
mocker.patch('src.utils.image_summaries.create_pair_summaries')
mocker.patch('src.estimator.training.training.train')
mocker.patch('src.estimator.launcher.providing_launcher.provide_launcher',
return_value=launcher)
undefined_commandline_flag = ("foo", 34)
defined_commandline_flag = (consts.BATCH_SIZE, 42)
pass_cli_arg([undefined_commandline_flag, defined_commandline_flag])
testing_utils.testing_helpers.run_app()
log = list(
filenames.get_run_text_logs_dir(
launcher.runs_data[0]).iterdir())[0].read_text()
lines = tuple(
open(
str(
list(
filenames.get_run_text_logs_dir(
launcher.runs_data[0]).iterdir())[0]), 'r'))
line_in_question = [
x for x in lines if 'Undefined commandline flags' in x
][0]
assert "--foo=34" in line_in_question
def test_should_provide_single_run_data_for_default_launcher(mocker):
model = FakeModel()
launcher = DefaultLauncher([model])
mocker.patch('src.estimator.launcher.providing_launcher.provide_launcher',
return_value=launcher)
result = providing_launcher.provide_single_run_data()
assert result == launcher.runs_data[0]
assert result.model == model
def test_should_create_correct_text_log_name(mocker, run_dir_mock):
mocker.patch('time.strftime', return_value=mocked_strftime)
summary = "summary"
get_run_summary_mock = mocker.patch('src.utils.utils.get_run_summary',
return_value=summary)
model = FakeModel()
dir_name = filenames.create_text_log_name(model)
assert_that(str(dir_name),
ends_with(summary + '_' + mocked_strftime + '.log'))
get_run_summary_mock.assert_called_once_with(model)
def test_create_pair_summaries(patched_dataset_reading):
provider = patched_dataset_reading.param
run_data = gen.run_data(model=FakeModel(data_provider=provider()))
dir_with_pair_summaries = filenames.get_run_logs_data_dir(
run_data) / 'features'
assert utils.check_filepath(dir_with_pair_summaries, exists=False)
image_summaries.create_pair_summaries(run_data)
assert utils.check_filepath(dir_with_pair_summaries,
is_directory=True,
is_empty=False)
assert len(list(dir_with_pair_summaries.iterdir())) == 1
def test_training_in_memory_only(mocker):
launcher = DefaultLauncher([FakeModel()])
before_run_mock, in_memory_training_mock = prepare_mocks(launcher, mocker)
distributed_training_mock = mocker.patch(
'src.estimator.training.training.distributed_train_eval',
autospec=True)
import sys
training.main(sys.argv)
before_run_mock.assert_called_once()
in_memory_training_mock.assert_called_once()
distributed_training_mock.assert_not_called()
def test_should_create_run_dir_for_default_launcher_ignoring_global_suffix(
mocker, patched_params):
launcher = DefaultLauncher([FakeModel()])
run_data: RunData = launcher.runs_data[0]
summary = "run_summary"
mocker.patch('src.estimator.launcher.providing_launcher.provide_launcher',
return_value=launcher)
get_run_summary_mock = mocker.patch('src.utils.utils.get_run_summary',
return_value=summary)
dir_name = filenames.get_run_dir(run_data)
assert_that(
str(dir_name),
ends_with('/tf/runs/{}/{}/{}'.format(run_data.runs_directory_name,
run_data.launcher_name, summary)))
get_run_summary_mock.assert_called_once_with(run_data.model)
def test_should_create_run_dir_for_experiment_launcher(mocker,
patched_global_suffix):
launcher = FakeExperimentLauncher([FakeModel()])
run_data: RunData = launcher.runs_data[0]
summary = "run_summary"
mocker.patch('src.estimator.launcher.providing_launcher.provide_launcher',
return_value=launcher)
get_run_summary_mock = mocker.patch('src.utils.utils.get_run_summary',
return_value=summary)
dir_name = filenames.get_run_dir(run_data)
assert_that(
str(dir_name),
ends_with('/tf/runs/{}/{}/{}'.format(
run_data.runs_directory_name, run_data.launcher_name +
(('_' + patched_global_suffix)
if patched_global_suffix is not None else ""), summary)))
get_run_summary_mock.assert_called_once_with(run_data.model)
def test_should_create_correct_run_summary(suffix, patched_global_suffix,
patched_excluded):
model = FakeModel()
dir_name = utils.get_run_summary(model)
assert dir_name.endswith(suffix)
inference.single_run_inference(run_data=run_data, show=False)
def test_should_not_throw_if_model_has_more_checkpoints():
run_data = gen.run_data(with_model_dir=True)
model_dir = filenames.get_run_logs_data_dir(run_data)
empty_checkpoints = [(model_dir / ("model.ckpt-{}.foobar".format(x)))
for x in range(2)]
[f.write_text("this is sparta") for f in empty_checkpoints]
inference.single_run_inference(run_data=run_data, show=False)
@pytest.mark.integration
@pytest.mark.parametrize('launcher', [
DefaultLauncher([FakeModel()]),
FakeExperimentLauncher([FakeModel(), FakeModel()]),
])
@pytest.mark.parametrize('patched_params',
[{
consts.IS_INFER_CHECKPOINT_OBLIGATORY: False
}],
indirect=True)
def test_should_run_inference_for_different_launchers(mocker, launcher,
patched_params):
mocker.patch('src.estimator.launcher.providing_launcher.provide_launcher',
return_value=launcher)
result = mocker.patch('builtins.input', return_value='0')
inference.infer()
if launcher.is_experiment:
result.assert_called_once()
def test_should_pass_model_dir_to_estimator():
model = FakeModel()
run_data = gen.run_data(model)
estimator = training.create_estimator(run_data)
model_dir = estimator.params[consts.MODEL_DIR]
assert model_dir == str(filenames.get_run_logs_data_dir(run_data))