def test_pipeline(self):
config = Config()
pipeline(config, config_file=TEST_CONFIG_PIPE)
assert config.dataprep['run'] == True
assert config.dataprep['job_dir'] == str(TEST_JOB_DIR)
assert config.dataprep['samples_file'] == str(TEST_SAMPLES)
assert config.dataprep['image_dir'] == TEST_IMAGE_DIR_RES
assert config.dataprep['resize'] == True
assert config.train['run'] == True
assert config.train['cloud'] == False
assert config.evaluate['run'] == True
assert config.cloud['run'] == False
assert config.cloud['provider'] == 'aws'
assert config.cloud['tf_dir'] == 'cloud/aws'
assert config.cloud['region'] == 'eu-west-1'
assert config.cloud['vpc_id'] == 'abc'
assert config.cloud['instance_type'] == 't2.micro'
assert config.cloud['bucket'] == 's3://test_bucket'
assert config.cloud['destroy'] == True
assert config.cloud['cloud_tag'] == 'test_user'
assert list(TEST_JOB_DIR.glob('*/confusion_matrix.pdf'))
assert list(TEST_JOB_DIR.glob('*/test_set_distribution.pdf'))
assert list(Path(TEST_JOB_DIR / 'models').glob('*.hdf5'))
def pipeline(config: Config, **kwargs):
"""Runs all components for which run=True in config file.
All activated (run=True) components from config file will be run in sequence. Options overwrite the config file.
The config file is the only way to define pipeline components.
Args:
config-file: Central configuration file.
"""
commands.pipeline(config, **kwargs)
def test_pipeline(self, mocker):
BEST_MODEL_FILE = list(Path(TEST_JOB_DIR /
'models').glob('*.hdf5'))[-1]
BEST_MODEL = 'evaluation_' + BEST_MODEL_FILE.stem
NB_FILEPATH = TEST_JOB_DIR / BEST_MODEL / 'evaluation_report.ipynb'
def fake_execute_notebook(*args, **kwargs):
shutil.copy(TEST_NB_TEMPLATE, NB_FILEPATH)
mocker.patch('papermill.execute_notebook',
side_effect=fake_execute_notebook)
mocker.patch(
'imageatm.components.evaluation.Evaluation._determine_best_modelfile',
return_value=BEST_MODEL_FILE)
mocker.patch('nbconvert.PDFExporter.from_notebook_node',
return_value=('ANY_DATA'.encode(), None))
config = Config()
pipeline(config, config_file=TEST_CONFIG_PIPE)
assert config.dataprep['run'] == True
assert config.dataprep['job_dir'] == str(TEST_JOB_DIR)
assert config.dataprep['samples_file'] == str(TEST_SAMPLES)
assert config.dataprep['image_dir'] == TEST_IMAGE_DIR_RES
assert config.dataprep['resize'] == True
assert config.train['run'] == True
assert config.train['cloud'] == False
assert config.evaluate['run'] == True
assert config.evaluate['report']['create'] == True
assert config.evaluate['report']['kernel_name'] == 'any_kernel'
assert config.evaluate['report']['export_html'] == True
assert config.evaluate['report']['export_pdf'] == True
assert config.cloud['run'] == False
assert config.cloud['provider'] == 'aws'
assert config.cloud['tf_dir'] == 'cloud/aws'
assert config.cloud['region'] == 'eu-west-1'
assert config.cloud['vpc_id'] == 'abc'
assert config.cloud['instance_type'] == 't2.micro'
assert config.cloud['bucket'] == 's3://test_bucket'
assert config.cloud['destroy'] == True
assert config.cloud['cloud_tag'] == 'test_user'
assert list(Path(TEST_JOB_DIR / 'models').glob('*.hdf5'))
def test_pipeline(mocker):
# assert that only dataprep gets run
TEST_CONFIG_FILE = p.resolve(
).parent / 'test_configs' / 'config_dataprep.yml'
config = Config()
m_dp, m_tc, m_t, m_e, m_c, m_l = mock_scripts(mocker)
pipeline(config, config_file=TEST_CONFIG_FILE)
m_dp.assert_called()
m_tc.assert_not_called()
m_t.assert_not_called()
m_e.assert_not_called()
m_c.assert_not_called()
# assert that only train gets run
TEST_CONFIG_FILE = p.resolve().parent / 'test_configs' / 'config_train.yml'
m_dp, m_tc, m_t, m_e, m_c, m_l = mock_scripts(mocker)
pipeline(config, config_file=TEST_CONFIG_FILE)
m_dp.assert_not_called()
m_tc.assert_not_called()
m_t.assert_called()
m_e.assert_not_called()
m_c.assert_not_called()
# assert that only train cloud gets run
TEST_CONFIG_FILE = p.resolve(
).parent / 'test_configs' / 'config_train_cloud.yml'
m_dp, m_tc, m_t, m_e, m_c, m_l = mock_scripts(mocker)
pipeline(config, config_file=TEST_CONFIG_FILE)
m_dp.assert_not_called()
m_tc.assert_called()
m_t.assert_not_called()
m_e.assert_not_called()
m_c.assert_not_called()
# assert that only evaluate gets run
TEST_CONFIG_FILE = p.resolve(
).parent / 'test_configs' / 'config_evaluate.yml'
m_dp, m_tc, m_t, m_e, m_c, m_l = mock_scripts(mocker)
pipeline(config, config_file=TEST_CONFIG_FILE)
m_dp.assert_not_called()
m_tc.assert_not_called()
m_t.assert_not_called()
m_e.assert_called()
m_c.assert_not_called()
# assert that only cloud gets run
TEST_CONFIG_FILE = p.resolve().parent / 'test_configs' / 'config_cloud.yml'
m_dp, m_tc, m_t, m_e, m_c, m_l = mock_scripts(mocker)
pipeline(config, config_file=TEST_CONFIG_FILE)
m_dp.assert_not_called()
m_tc.assert_not_called()
m_t.assert_not_called()
m_e.assert_not_called()
m_c.assert_called()
# assert that all components get run
TEST_CONFIG_FILE = p.resolve().parent / 'test_configs' / 'config_all.yml'
m_dp, m_tc, m_t, m_e, m_c, m_l = mock_scripts(mocker)
pipeline(config, config_file=TEST_CONFIG_FILE)
m_dp.assert_called()
m_tc.assert_not_called()
m_t.assert_called()
m_e.assert_called()
m_c.assert_called()