def test_PipelineRunFailed_raised_when_subprocess_failed(
self, mock_popen, *args):
mock_popen.return_value.poll.return_value = -1
mock_popen.return_value.return_code = -1
executor = LocalPipelineExecutor()
self.assertRaises(PipelineRunFailed, executor.run_pipeline_collection,
self.pipeline)
def test_Popen_is_called_when_command_executes(self, mock_popen, *args):
executor = LocalPipelineExecutor()
command = 'hello'
executor.execute_command(command, watch=False)
mock_popen.assert_called_with(command)
executor.execute_command(command, job_name='job', watch=True)
mock_popen.assert_called_with(command, shell=True)
def test_process_saved_when_command_called_with_watch(
self, mock_popen, *args):
executor = LocalPipelineExecutor()
command = 'hello'
job_name = 'name'
executor.execute_command(command, watch=True, job_name=job_name)
mock_popen.assert_called_with(command, shell=True)
self.assertIn(job_name, executor.running_jobs)
self.assertIs(executor.running_jobs[job_name], mock_popen.return_value)
def test_job_are_finished_when_return_code_is_zero(self, mock_popen,
mock_sleep, *args):
mock_popen.return_value.poll.return_value = 0
mock_popen.return_value.returncode = 0
executor = LocalPipelineExecutor()
executor._parse_results_file = lambda *args, **kwargs: '1'
executor.run_pipeline_collection(self.pipeline)
self.assertEqual(len(executor.running_jobs), 0)
self.assertFalse(mock_sleep.called)
def test_Popen_is_not_called_with_bad_command(self, mock_popen, *args):
executor = LocalPipelineExecutor()
bad_commands = [
'',
0,
True,
['Commands'],
{
'command': 'again'
},
' ',
'\n',
'\t',
]
for bad_command in bad_commands:
self.assertRaises(ValueError, executor.execute_command,
bad_command)
self.assertFalse(mock_popen.called)
def test_jobs_are_polled_agained_when_poll_is_None(self, mock_popen,
*args):
polled = {'yes': False, 'calls': 0}
def poll():
polled['calls'] += 1
if polled['yes']:
mock_popen.return_value.returncode = 0
return 0
else:
polled['yes'] = True
return None
mock_popen.return_value.poll = poll
executor = LocalPipelineExecutor(run_serial=False)
executor._parse_results_file = lambda *args, **kwargs: '1'
executor.run_pipeline_collection(self.pipeline)
# Called twice for first step and once for second.
self.assertGreater(polled['calls'], 0)
class TestLocalParallelRun(TestLocalSerialRun):
executor = lambda *a, **kw: LocalPipelineExecutor(
*a, run_serial=False, poll_interval=1, **kw)
from doepipeline.designer import BaseExperimentDesigner
import pandas as pd
import numpy as np
class ExampleDesigner(BaseExperimentDesigner):
def __init__(self, *args, **kwargs):
super(ExampleDesigner, self).__init__(*args, **kwargs)
self.design = None
np.random.seed(123456789)
def update_factors_from_response(self, response):
self.design += 1
return self.design
def new_design(self, factor_settings=None):
data = np.random.randint(10, size=(2, len(self.factors)))
self.design = pd.DataFrame(data,
index=['A', 'B'],
columns=self.factors.keys())
return self.design
if __name__ == '__main__':
generator = PipelineGenerator.from_yaml('example_pipeline.yaml')
designer = generator.new_designer_from_config(ExampleDesigner)
design = designer.new_design()
pipeline = generator.new_pipeline_collection(design)
executor = LocalPipelineExecutor()
results = executor.run_pipeline_collection(pipeline)
design = designer.update_factors_from_response(results)