def setUp(self): super(TestMakePipeline, self).setUp() self.env_vars = {key: value for key, value in \ self.config['before_run']['environment_variables'].items()} self.runner = PipelineGenerator(self.config) self.scripts1 = ['./script_a --option 0.1', './script_b 0.2'] self.scripts2 = ['./script_a --option 0.3', './script_b 0.4']
def test_run_optimum_within_bounds(self): generator = PipelineGenerator(self.config_with_optimum(15, 3)) designer = generator.new_designer_from_config( model_selection='manual', manual_formula=self.full_formula) design = designer.new_design() pipeline = generator.new_pipeline_collection(design) cmd = '{script}' executor = self.__class__.executor(workdir=self.work_dir, base_command=cmd) results = executor.run_pipeline_collection(pipeline) optimum = designer.update_factors_from_response(results) expected_optimum = {'FactorA': 15, 'FactorB': 3} self.assertTrue(optimum.converged) for factor in design.columns: self.assertTrue( np.isclose(optimum.predicted_optimum[factor], expected_optimum[factor]))
def test_screening_finds_optimum_at_origin(self): generator = PipelineGenerator(self.config_with_optimum(0, 0)) designer = generator.new_designer_from_config( skip_screening=False, model_selection='manual', manual_formula=self.full_formula) design = designer.new_design() pipeline = generator.new_pipeline_collection(design) cmd = '{script}' executor = self.__class__.executor(workdir=self.work_dir, base_command=cmd) results = executor.run_pipeline_collection(pipeline) optimum = designer.update_factors_from_response(results) self.assertEqual(designer._phase, 'optimization') self.assertFalse(optimum.converged) expected_optimum = {'FactorA': 0, 'FactorB': 0} for factor in design.columns: self.assertTrue( np.isclose(optimum.predicted_optimum[factor], expected_optimum[factor]))
class TestMakePipeline(BaseGeneratorTestCase): def setUp(self): super(TestMakePipeline, self).setUp() self.env_vars = {key: value for key, value in \ self.config['before_run']['environment_variables'].items()} self.runner = PipelineGenerator(self.config) self.scripts1 = ['./script_a --option 0.1', './script_b 0.2'] self.scripts2 = ['./script_a --option 0.3', './script_b 0.4'] def test_render_experiments_without_id_column(self): pipeline_collection = self.runner.new_pipeline_collection( self.dummy_design) expected = { 0: self.scripts1, 1: self.scripts2, 'ENV_VARIABLES': self.env_vars, 'SETUP_SCRIPTS': None, 'RESULTS_FILE': self.config['results_file'], 'WORKDIR': self.config['working_directory'], 'JOBNAMES': ['ScriptWithOptions', 'ScriptWithSub'] } self.assertDictEqual(expected, pipeline_collection) def test_render_experiments_with_id_column(self): new_collection = self.runner.new_pipeline_collection( self.dummy_design, 'Exp Id') self.maxDiff = None expected = { 'A': self.scripts1, 'B': self.scripts2, 'ENV_VARIABLES': self.env_vars, 'SETUP_SCRIPTS': None, 'RESULTS_FILE': self.config['results_file'], 'WORKDIR': self.config['working_directory'], 'JOBNAMES': ['ScriptWithOptions', 'ScriptWithSub'] } self.assertDictEqual(expected, new_collection)
def test_run_optimum_outside_bounds(self): generator = PipelineGenerator(self.config_with_optimum(5, 5)) designer = generator.new_designer_from_config( model_selection='manual', manual_formula=self.full_formula) design = designer.new_design() pipeline = generator.new_pipeline_collection(design) cmd = '{script}' executor = self.__class__.executor(workdir=self.work_dir, base_command=cmd) results = executor.run_pipeline_collection(pipeline) optimum = designer.update_factors_from_response(results) expected_optimum = {'FactorA': 5, 'FactorB': 5} self.assertFalse(optimum.converged) for factor in design.columns: # Optimum is outside of bounds and optimization should be # constrained to be inside design space. So the expected # optimum should not have been predicted after a single # iteration. self.assertFalse( np.isclose(optimum.predicted_optimum[factor], expected_optimum[factor]))
from doepipeline.designer import BaseExperimentDesigner import pandas as pd import numpy as np class ExampleDesigner(BaseExperimentDesigner): def __init__(self, *args, **kwargs): super(ExampleDesigner, self).__init__(*args, **kwargs) self.design = None np.random.seed(123456789) def update_factors_from_response(self, response): self.design += 1 return self.design def new_design(self, factor_settings=None): data = np.random.randint(10, size=(2, len(self.factors))) self.design = pd.DataFrame(data, index=['A', 'B'], columns=self.factors.keys()) return self.design if __name__ == '__main__': generator = PipelineGenerator.from_yaml('example_pipeline.yaml') designer = generator.new_designer_from_config(ExampleDesigner) design = designer.new_design() pipeline = generator.new_pipeline_collection(design) executor = LocalPipelineExecutor() results = executor.run_pipeline_collection(pipeline) design = designer.update_factors_from_response(results)
def setUp(self): script_one = { 'script': './script_a', 'factors': { 'FactorA': { 'factor_name': 'Factor A', 'script_option': '--option' } } } script_two = { 'script': './script_b {% FactorB %}', 'factors': { 'FactorB': { 'factor_name': 'Factor B', 'substitute': True } } } design_spec = { 'type': 'CCC', 'factors': { 'FactorA': { 'min': 0, 'max': 1, 'low_init': 0, 'high_init': .2 }, 'FactorB': { 'min': 0, 'max': 1, 'low_init': .1, 'high_init': .3 } }, 'responses': { 'ResponseA': {'criterion': 'maximize'} } } self.work_dir = os.path.join(os.getcwd(), 'work_dir') os.makedirs(self.work_dir, exist_ok=True) self.env_vars = {'MYPATH': '~/a/path'} before = {'environment_variables': self.env_vars} self.outfile = 'my_results.txt' self.config = { 'working_directory': self.work_dir, 'design': design_spec, 'results_file': self.outfile, 'before_run': before, 'pipeline': ['ScriptOne', 'ScriptTwo'], 'ScriptOne': script_one, 'ScriptTwo': script_two } self.design = pd.DataFrame([ ['One', .1, .2], ['Two', .3, .4] ], columns=['Exp Id', 'FactorA', 'FactorB']) self.generator = PipelineGenerator(copy.deepcopy(self.config)) self.pipeline = self.generator.new_pipeline_collection(self.design,'Exp Id')
class ExecutorTestCase(unittest.TestCase): executor_class = MockBaseExecutor init_args = tuple() init_kwargs = dict() def setUp(self): script_one = { 'script': './script_a', 'factors': { 'FactorA': { 'factor_name': 'Factor A', 'script_option': '--option' } } } script_two = { 'script': './script_b {% FactorB %}', 'factors': { 'FactorB': { 'factor_name': 'Factor B', 'substitute': True } } } design_spec = { 'type': 'CCC', 'factors': { 'FactorA': { 'min': 0, 'max': 1, 'low_init': 0, 'high_init': .2 }, 'FactorB': { 'min': 0, 'max': 1, 'low_init': .1, 'high_init': .3 } }, 'responses': { 'ResponseA': {'criterion': 'maximize'} } } self.work_dir = os.path.join(os.getcwd(), 'work_dir') os.makedirs(self.work_dir, exist_ok=True) self.env_vars = {'MYPATH': '~/a/path'} before = {'environment_variables': self.env_vars} self.outfile = 'my_results.txt' self.config = { 'working_directory': self.work_dir, 'design': design_spec, 'results_file': self.outfile, 'before_run': before, 'pipeline': ['ScriptOne', 'ScriptTwo'], 'ScriptOne': script_one, 'ScriptTwo': script_two } self.design = pd.DataFrame([ ['One', .1, .2], ['Two', .3, .4] ], columns=['Exp Id', 'FactorA', 'FactorB']) self.generator = PipelineGenerator(copy.deepcopy(self.config)) self.pipeline = self.generator.new_pipeline_collection(self.design,'Exp Id') @mock.patch('os.makedirs') def test_execute_commands_returns_tuple(self, *args): executor = self.executor_class(*self.init_args, **self.init_kwargs) result = executor.execute_command('test') self.assertIsInstance(result, Sequence) self.assertEqual(len(result), 3) for value in result: self.assertIsInstance(value, str)
def test_bad_config_raises_valueerror(self): # Create bad pipeline by "skipping" one step in pipeline bad_config = copy.deepcopy(self.config) bad_config['pipeline'].pop() self.assertRaises(ValueError, lambda: PipelineGenerator(bad_config))
def test_bad_yaml_raises_valueerror(self): self.assertRaises(ValueError, lambda: PipelineGenerator.from_yaml({}))
def test_creating_from_yaml_doesnt_crash(self): yaml_generator = PipelineGenerator.from_yaml(self.yaml_path) with open(self.yaml_path) as f: buffer_generator = PipelineGenerator.from_yaml(f)
def test_creating_doesnt_crash(self): generator = PipelineGenerator(self.config)