def test_run(): w = Wait(seconds=0.1) executor = Executor(dependencies=[w], scan_interval=0.01) start_time = time.time() executor.execute() assert time.time() - start_time > 0.1
def compile(self, config): if COMPILERS_KEY in config: compilers = config.pop(COMPILERS_KEY) if isinstance(compilers, list): compilers = self._make_linear_chain(compilers, config) elif not isinstance(compilers, dict): raise TypeError( '{!r} key must specify either a list or a dict of compilers' .format(COMPILERS_KEY)) compiler_to_run_from = None for compiler_name, compiler_config in compilers.items(): if compiler_config.get(self.RUN_FROM_HERE, False): assert compiler_to_run_from is None, "Only one compiler can specify '{!s}', but both {!r} and {!r} specify it".format( self.RUN_FROM_HERE, compiler_to_run_from, compiler_name) compiler_to_run_from = compiler_name compiler_config.setdefault('input_step', self.INPUT_STEP_NAME) compilers[self.INPUT_STEP_NAME] = { 'class': 'daisychain.steps.input.InMemoryInput', 'output': config } instantiator = Instantiator(name=self.name, config=compilers) instantiator.run() executor = Executor(name=self.name, dependencies=list(instantiator.steps.values())) executor.execute() if compiler_to_run_from: return instantiator.steps[compiler_to_run_from].output else: return config
def test_run(): w = ThreadedWait(seconds=0.5) w2 = ThreadedWait(seconds=0.5) executor = Executor(dependencies=[w, w2], scan_interval=0.01) start_time = time.time() executor.execute() assert 0.5 < time.time() - start_time < 1.0
def compile(self, config): if COMPILERS_KEY in config: compilers = config.pop(COMPILERS_KEY) if isinstance(compilers, list): compilers = self._make_linear_chain(compilers, config) elif not isinstance(compilers, dict): raise TypeError('{!r} key must specify either a list or a dict of compilers'.format(COMPILERS_KEY)) compiler_to_run_from = None for compiler_name, compiler_config in compilers.items(): if compiler_config.get(self.RUN_FROM_HERE, False): assert compiler_to_run_from is None, "Only one compiler can specify '{!s}', but both {!r} and {!r} specify it".format(self.RUN_FROM_HERE, compiler_to_run_from, compiler_name) compiler_to_run_from = compiler_name compiler_config.setdefault('input_step', self.INPUT_STEP_NAME) compilers[self.INPUT_STEP_NAME] = {'class': 'daisychain.steps.input.InMemoryInput', 'output': config} instantiator = Instantiator(name=self.name, config=compilers) instantiator.run() executor = Executor(name=self.name, dependencies=list(instantiator.steps.values())) executor.execute() if compiler_to_run_from: return instantiator.steps[compiler_to_run_from].output else: return config
def test_manual_step(): m = Manual(instructions='Do something') e = Executor(name='test_manual', dependencies=[m]) with patch(input_function) as mock_raw_input: mock_raw_input.side_effect = ['a'] e.execute() assert m.failed
def test_execute_dry_run(): dep = test_step.MockStep(name='dep') assert not dep.checked_ready assert dep.status.pending e = Executor(dry_run=True, dependencies=[dep]) e.execute() assert dep.status.validated assert not dep.ran_once
def test_execute_dry_run(): dep = test_step.MockStep(name='dep') assert not dep.checked_ready assert dep.status.pending e = Executor(dry_run=True,dependencies=[dep]) e.execute() assert dep.status.validated assert not dep.ran_once
def test_execute_check_status_failure_in_step(): dep = test_step.MockStep(name='mock_step', check_status_exception=TypeError("Exception while checking status")) e = Executor(name='test_executor', dependencies=[dep]) try: e.execute() except CheckStatusException: assert dep.failed else: assert False, "Should have thrown a CheckStatusException on failure" dep = test_step.MockStep(name='mock_step', check_status_exception=TypeError("Exception while checking status")) dep2 = test_step.MockStep(name='mock_failing_step_parent', dependencies=[dep]) e = Executor(name='test_executor', dependencies=[dep2]) try: e.execute() except CheckStatusException: assert dep2.validated assert dep.failed else: assert False, "Should have thrown a CheckStatusException on failure" dep = test_step.MockStep(name='mock_step') def raise_error(): raise RuntimeError("Exception while forwarding callback") dep.status.check = raise_error dep2 = test_step.MockStep(name='mock_failing_step_parent', dependencies=[dep]) e = Executor(name='test_executor', dependencies=[dep2]) try: e.execute() except CheckStatusException: assert dep2.validated assert dep.failed else: assert False, "Should have thrown a CheckStatusException on failure"
def test_attach_self_as_executor(): e = Executor(name='test_executor') assert e.scan_interval == 0.0 assert e.execution is None assert e.user_input_class is ConsoleInput assert e.on_failure is Executor.RAISE dep = test_step.MockStep(name='mock_step') e.dependencies.add(dep) e.execution = Execution(executor=e) assert dep.executor is e assert dep.root_log_id == e.root_log_id
def run(self): instantiation_steps = self.get_instantiation_steps() self.log('instantiator').debug("Linking reference dependencies") for step in list(instantiation_steps.values()): step.evaluate_reference_dependencies(instantiation_steps) self.log('instantiator').debug("Running instantiator Tree") instantiation_executor = Executor(name=self.name) if self.root_log_id: instantiation_executor.root_log_id = self.root_log_id + '.' + instantiation_executor.root_log_id instantiation_executor.dependencies.update(set(instantiation_steps.values())) self.executor_output = instantiation_executor.execute() self.status.set_finished()
def compile(self, config): parenting_steps = dict() executor = Executor(name=self.name) for step_name, step_config in config[STEPS_KEY].items(): parenting_steps[step_name] = StepConfigInheritance.ParentedConfigurationStep(step_config, name=step_name) executor.dependencies.update(set(parenting_steps.values())) for parenting_step in list(parenting_steps.values()): parenting_step.evaluate_parentage(parenting_steps) executor.execute() return config
def run(self): instantiation_steps = self.get_instantiation_steps() self.log('instantiator').debug("Linking reference dependencies") for step in list(instantiation_steps.values()): step.evaluate_reference_dependencies(instantiation_steps) self.log('instantiator').debug("Running instantiator Tree") instantiation_executor = Executor(name=self.name) if self.root_log_id: instantiation_executor.root_log_id = self.root_log_id + '.' + instantiation_executor.root_log_id instantiation_executor.dependencies.update( set(instantiation_steps.values())) self.executor_output = instantiation_executor.execute() self.status.set_finished()
def test_execute_skip_failures(): dep = test_step.MockStep(name='mock_sibling_step', run_exception=RuntimeError("test_run_exception")) dep2 = test_step.MockStep(name='mock_sibling_step2', run_exception=RuntimeError("test_run_exception")) successful_dep = test_step.MockStep(name='successful_dep') parent = test_step.MockStep(name='mock_parent_step', dependencies=[dep, dep2, successful_dep]) assert parent.dependencies == {dep, dep2, successful_dep} successful_parent = test_step.MockStep(name='mock_successful_parent', dependencies=[successful_dep]) e = Executor(name='test_executor', on_failure=Executor.SKIP, dependencies=[parent, successful_parent]) e.execute() assert dep.failed assert dep2.failed assert successful_dep.finished assert parent.validated assert successful_parent.finished assert not e.execution.aborted
def test_cannot_find_command(): with patch('subprocess.Popen') as MockPopen: instance = MockPopen.returnvalue instance.returncode = 1 c = RunCommand(command='ls -l', poll_interval_seconds=0) executor = Executor(dependencies=[c]) try: executor.execute() except RuntimeError: pass except Exception as e: assert False, "Exception was supposed to be a RuntimeError but was {!r} instead".format(e) else: assert False, "Exception was not raised".format(e)
def test_execute_graceful_shutdown(): dep = test_step.MockStep(name='mock_sibling_step', run_exception=RuntimeError("test_run_exception")) dep2 = test_step.MockStep(name='mock_sibling_step2') dep2.run = lambda: dep2.status.set_validated() successful_dep = test_step.MockStep(name='successful_dep') parent = test_step.MockStep(name='mock_parent_step', dependencies=[dep, dep2, successful_dep]) assert parent.dependencies == {dep, dep2, successful_dep} successful_parent = test_step.MockStep(name='mock_successful_parent', dependencies=[successful_dep]) e = Executor(name='test_executor', on_failure=Executor.GRACEFUL_SHUTDOWN, dependencies=[parent, successful_parent]) e.execute() assert dep.status.failed assert dep2.status.finished or dep2.status.validated assert successful_dep.status.finished or successful_dep.status.validated assert parent.validated assert successful_parent.validated assert e.execution.aborted
def test_execute_graceful_shutdown_with_already_aborted_execution(): dep = test_step.MockStep(name='mock_sibling_step') dep2 = test_step.MockStep(name='mock_sibling_step2', run_exception=RuntimeError("test_run_exception")) successful_dep = test_step.MockStep(name='successful_dep') parent = test_step.MockStep(name='mock_parent_step', dependencies=[dep, dep2, successful_dep]) assert parent.dependencies == {dep, dep2, successful_dep} successful_parent = test_step.MockStep(name='mock_successful_parent', dependencies=[successful_dep]) e = Executor(name='test_executor', on_failure=Executor.GRACEFUL_SHUTDOWN, dependencies=[parent, successful_parent]) e.execution = Execution(executor=e) e.execution.aborted = True e.execute() assert dep.status.pending assert dep2.status.pending assert successful_dep.status.pending assert parent.status.pending assert successful_parent.status.pending
def test_prompt_user_for_step(): with patch(input_function) as mock_raw_input: dep = test_step.MockStep(name='mock_step', run_exception=RuntimeError('Exception while running step')) e = Executor(name='test_executor') global times_called times_called = 0 def raw_input_output(*args, **kwargs): global times_called times_called += 1 responses = ['y','','r',''] prompt = kwargs.get('prompt', args[0]) assert 'mock_step' in prompt assert 'Does this test work (y)/(n)?' in prompt return responses[times_called - 1] mock_raw_input.side_effect = raw_input_output assert e.prompt_user_for_step(step=dep, prompt='Does this test work (y)/(n)?') == 'y' assert e.prompt_user_for_step(step=dep, prompt='Does this test work (y)/(n)?', valid_choices=['d','r']) == 'r' assert e.prompt_user_for_step(step=dep, prompt='Does this test work (y)/(n)?', default='n') == 'n' e.execution = Execution() e.execution.aborted = True try: e.prompt_user_for_step(step=dep, prompt='Does this test work (y)/(n)?') except ExecutorAborted: pass else: assert False, 'Should have raised an ExecutorAborted exception if it was previously aborted'
def test_prompting_during_execution(): with patch(input_function) as mock_raw_input: dep = test_step.MockStep(name='mock_step', run_exception=RuntimeError('Exception while running step')) e = Executor(name='test_executor', on_failure=Executor.PROMPT, dependencies=[dep]) global times_called times_called = 0 def raw_input_output(*args, **kwargs): global times_called times_called += 1 if times_called == 1: return 'r' elif times_called == 2: return 'f' assert times_called <= 2, "Called raw_input too many times" mock_raw_input.side_effect = raw_input_output e.execute() assert dep.status.finished assert times_called == 2
def test_init(): e = Executor() assert e.scan_interval == 0.0 assert e.execution is None assert e.user_input_class is ConsoleInput assert e.on_failure is Executor.RAISE e = Executor(on_failure=Executor.PROMPT, scan_interval=1.0) assert e.scan_interval == 1.0 assert e.execution is None assert e.user_input_class is ConsoleInput assert e.on_failure is Executor.PROMPT try: e = Executor(on_failure='NOT_A_KNOWN_FAILURE_TYPE') except ValueError: pass else: assert False, "Should have thrown a Value Error for an unknown failure mode"
def test_failing_command(): with patch('subprocess.Popen') as MockPopen: validate_mock, command_mock = MagicMock(spec=RealPopen)(), MagicMock(spec=RealPopen)() MockPopen.side_effect = [validate_mock, command_mock] validate_mock.returncode = 0 command_mock.poll.return_value = 1 stdin = command_mock.stdin.return_value stdin.write.return_value = None stdin.close.return_value = None c = RunCommand(command='ls -l', poll_interval_seconds=0) executor = Executor(dependencies=[c]) try: executor.execute() except subprocess.CalledProcessError: pass except Exception as e: assert False, "Exception was supposed to be a CalledProcessError but was {!r} instead".format(e) else: assert False, "Exception was not raised".format(e)
def test_prompt_user_for_status(): s = MockStep() s.prompt_user_for_status() assert s.status.failed assert isinstance(s.status.stage, RuntimeError) def mock_prompt_user(response): def wrapper(*args, **kwargs): return response return wrapper s.executor = Executor() s.executor.execution = Execution() s.prompt_user = mock_prompt_user('f') s.prompt_user_for_status() assert s.status.finished s.prompt_user = mock_prompt_user('r') s.prompt_user_for_status() assert s.status.pending s.prompt_user = mock_prompt_user('a') s.prompt_user_for_status() assert s.status.failed assert isinstance(s.status.stage, RuntimeError) s.status.set_failed(TypeError("A different kind of error")) s.prompt_user_for_status(exception=s.status.stage) assert isinstance(s.status.stage, TypeError) s.prompt_user = mock_prompt_user('r') s.status.stage = CheckStatusException(s.status, previous_stage='mock_stage', exception=TypeError("Error during check status")) s.prompt_user_for_status(exception=s.status.stage) assert s.status.stage == 'mock_stage' s.prompt_user = mock_prompt_user('a') s.status.stage = CheckStatusException(s.status, previous_stage='mock_stage', exception=TypeError("Error during check status")) s.prompt_user_for_status(exception=s.status.stage) assert s.status.failed assert isinstance(s.status.stage, CheckStatusException) s.prompt_user = mock_prompt_user('f') s.status.stage = CheckStatusException(s.status, previous_stage='mock_stage', exception=TypeError("Error during check status")) s.prompt_user_for_status(exception=s.status.stage) assert s.status.stage != 'mock_stage' assert s.status.finished def mock_prompt_user_error(*args, **kwargs): raise TypeError("This error came from prompt_user") s.prompt_user = mock_prompt_user_error s.prompt_user_for_status() assert isinstance(s.status.stage, TypeError)
def test_failing_auth_command(): with patch('subprocess.Popen') as MockPopen: validate_mock, auth_validate_mock = MagicMock(spec=RealPopen)(), MagicMock(spec=RealPopen)() MockPopen.side_effect = [validate_mock, auth_validate_mock] validate_mock.returncode = 0 validate_mock.communicate.return_value = (None, None) auth_validate_mock.returncode = 1 auth_validate_mock.communicate.return_value = (None, None) auth = BasicAuth(username='******', password='******') c = RunCommand(command='sudo -k mount -a', authentication=auth, poll_interval_seconds=0) executor = Executor(dependencies=[c]) try: executor.execute() except RuntimeError: pass except Exception as e: assert False, "Exception was supposed to be a RuntimeError but was {!r} instead".format(e) else: assert False, "Exception was not raised".format(e) assert validate_mock.communicate.call_count == 1 auth_validate_mock.communicate.assert_called_once_with('mockpassword\n')
def test_auth_command(): with patch('subprocess.Popen') as MockPopen: validate_mock, auth_validate_mock, command_mock = MagicMock(spec=RealPopen)(), MagicMock(spec=RealPopen)(), MagicMock(spec=RealPopen)() MockPopen.side_effect = [validate_mock, auth_validate_mock, command_mock] validate_mock.returncode = 0 validate_mock.communicate.return_value = (None, None) auth_validate_mock.returncode = 0 auth_validate_mock.communicate.return_value = (None, None) command_mock.poll.return_value = 0 stdin = command_mock.stdin.return_value stdin.write.return_value = None stdin.close.return_value = None auth = BasicAuth(username='******', password='******') c = RunCommand(command='sudo -k mount -a', authentication=auth, poll_interval_seconds=0) executor = Executor(dependencies=[c]) executor.execute() assert validate_mock.communicate.call_count == 1 auth_validate_mock.communicate.assert_called_once_with('mockpassword\n') assert command_mock.poll.call_count == 1 command_mock.stdin.write.assert_called_once_with('mockpassword\n') assert command_mock.stdin.close.call_count == 1
def test_prompting_during_execution(): with patch(input_function) as mock_raw_input: dep = test_step.MockStep( name='mock_step', run_exception=RuntimeError('Exception while running step')) e = Executor(name='test_executor', on_failure=Executor.PROMPT, dependencies=[dep]) global times_called times_called = 0 def raw_input_output(*args, **kwargs): global times_called times_called += 1 if times_called == 1: return 'r' elif times_called == 2: return 'f' assert times_called <= 2, "Called raw_input too many times" mock_raw_input.side_effect = raw_input_output e.execute() assert dep.status.finished assert times_called == 2
def test_prompt_user_for_step(): with patch(input_function) as mock_raw_input: dep = test_step.MockStep( name='mock_step', run_exception=RuntimeError('Exception while running step')) e = Executor(name='test_executor') global times_called times_called = 0 def raw_input_output(*args, **kwargs): global times_called times_called += 1 responses = ['y', '', 'r', ''] prompt = kwargs.get('prompt', args[0]) assert 'mock_step' in prompt assert 'Does this test work (y)/(n)?' in prompt return responses[times_called - 1] mock_raw_input.side_effect = raw_input_output assert e.prompt_user_for_step( step=dep, prompt='Does this test work (y)/(n)?') == 'y' assert e.prompt_user_for_step(step=dep, prompt='Does this test work (y)/(n)?', valid_choices=['d', 'r']) == 'r' assert e.prompt_user_for_step(step=dep, prompt='Does this test work (y)/(n)?', default='n') == 'n' e.execution = Execution() e.execution.aborted = True try: e.prompt_user_for_step(step=dep, prompt='Does this test work (y)/(n)?') except ExecutorAborted: pass else: assert False, 'Should have raised an ExecutorAborted exception if it was previously aborted'
def test_execute(): dep = test_step.MockStep(name='mock_step') e = Executor(name='test_executor', dependencies=[dep]) e.execute() assert dep.finished dep_named = test_step.MockStep(name='mock_step_named') dep = test_step.MockStep(name='mock_step', named_reference=dep_named) assert dep.named_reference is dep_named e = Executor(name='test_executor', dependencies=[dep]) e.execute() assert dep_named.finished assert dep.finished dep_dep = test_step.MockStep(name='mock_step_dep') dep = test_step.MockStep(name='mock_step', dependencies=[dep_dep]) assert dep.dependencies == {dep_dep} e = Executor(name='test_executor', dependencies=[dep]) assert e.dependencies == {dep} e.execute() assert dep_named.finished assert dep.finished dep = test_step.MockStep(name='mock_step', run_exception=RuntimeError('Exception while running step')) e = Executor(name='test_executor', dependencies=[dep]) try: e.execute() except RuntimeError: assert dep.failed else: assert False, "Should have thrown the error the step raised"
def test_execute_check_status_failure_in_step(): dep = test_step.MockStep( name='mock_step', check_status_exception=TypeError("Exception while checking status")) e = Executor(name='test_executor', dependencies=[dep]) try: e.execute() except CheckStatusException: assert dep.failed else: assert False, "Should have thrown a CheckStatusException on failure" dep = test_step.MockStep( name='mock_step', check_status_exception=TypeError("Exception while checking status")) dep2 = test_step.MockStep(name='mock_failing_step_parent', dependencies=[dep]) e = Executor(name='test_executor', dependencies=[dep2]) try: e.execute() except CheckStatusException: assert dep2.validated assert dep.failed else: assert False, "Should have thrown a CheckStatusException on failure" dep = test_step.MockStep(name='mock_step') def raise_error(): raise RuntimeError("Exception while forwarding callback") dep.status.check = raise_error dep2 = test_step.MockStep(name='mock_failing_step_parent', dependencies=[dep]) e = Executor(name='test_executor', dependencies=[dep2]) try: e.execute() except CheckStatusException: assert dep2.validated assert dep.failed else: assert False, "Should have thrown a CheckStatusException on failure"
def test_basic_command(): c = RunCommand(command='ls -l', poll_interval_seconds=0) assert 'ls -l' in c.instructions executor = Executor(dependencies=[c]) executor.execute()
def test_execute(): dep = test_step.MockStep(name='mock_step') e = Executor(name='test_executor', dependencies=[dep]) e.execute() assert dep.finished dep_named = test_step.MockStep(name='mock_step_named') dep = test_step.MockStep(name='mock_step', named_reference=dep_named) assert dep.named_reference is dep_named e = Executor(name='test_executor', dependencies=[dep]) e.execute() assert dep_named.finished assert dep.finished dep_dep = test_step.MockStep(name='mock_step_dep') dep = test_step.MockStep(name='mock_step', dependencies=[dep_dep]) assert dep.dependencies == {dep_dep} e = Executor(name='test_executor', dependencies=[dep]) assert e.dependencies == {dep} e.execute() assert dep_named.finished assert dep.finished dep = test_step.MockStep( name='mock_step', run_exception=RuntimeError('Exception while running step')) e = Executor(name='test_executor', dependencies=[dep]) try: e.execute() except RuntimeError: assert dep.failed else: assert False, "Should have thrown the error the step raised"