def compile(self, config): if COMPILERS_KEY in config: compilers = config.pop(COMPILERS_KEY) if isinstance(compilers, list): compilers = self._make_linear_chain(compilers, config) elif not isinstance(compilers, dict): raise TypeError('{!r} key must specify either a list or a dict of compilers'.format(COMPILERS_KEY)) compiler_to_run_from = None for compiler_name, compiler_config in compilers.items(): if compiler_config.get(self.RUN_FROM_HERE, False): assert compiler_to_run_from is None, "Only one compiler can specify '{!s}', but both {!r} and {!r} specify it".format(self.RUN_FROM_HERE, compiler_to_run_from, compiler_name) compiler_to_run_from = compiler_name compiler_config.setdefault('input_step', self.INPUT_STEP_NAME) compilers[self.INPUT_STEP_NAME] = {'class': 'daisychain.steps.input.InMemoryInput', 'output': config} instantiator = Instantiator(name=self.name, config=compilers) instantiator.run() executor = Executor(name=self.name, dependencies=list(instantiator.steps.values())) executor.execute() if compiler_to_run_from: return instantiator.steps[compiler_to_run_from].output else: return config
def test_run(): w = ThreadedWait(seconds=0.5) w2 = ThreadedWait(seconds=0.5) executor = Executor(dependencies=[w, w2], scan_interval=0.01) start_time = time.time() executor.execute() assert 0.5 < time.time() - start_time < 1.0
def test_manual_step(): m = Manual(instructions='Do something') e = Executor(name='test_manual', dependencies=[m]) with patch(input_function) as mock_raw_input: mock_raw_input.side_effect = ['a'] e.execute() assert m.failed
def test_run(): w = Wait(seconds=0.1) executor = Executor(dependencies=[w], scan_interval=0.01) start_time = time.time() executor.execute() assert time.time() - start_time > 0.1
def test_run(): w = ThreadedWait(seconds=0.5) w2 = ThreadedWait(seconds=0.5) executor = Executor(dependencies=[w, w2], scan_interval=0.01) start_time = time.time() executor.execute() assert 0.5 < time.time() - start_time < 1.0
def test_run(): w = Wait(seconds=0.1) executor = Executor(dependencies=[w], scan_interval=0.01) start_time = time.time() executor.execute() assert time.time() - start_time > 0.1
def compile(self, config): if COMPILERS_KEY in config: compilers = config.pop(COMPILERS_KEY) if isinstance(compilers, list): compilers = self._make_linear_chain(compilers, config) elif not isinstance(compilers, dict): raise TypeError( '{!r} key must specify either a list or a dict of compilers' .format(COMPILERS_KEY)) compiler_to_run_from = None for compiler_name, compiler_config in compilers.items(): if compiler_config.get(self.RUN_FROM_HERE, False): assert compiler_to_run_from is None, "Only one compiler can specify '{!s}', but both {!r} and {!r} specify it".format( self.RUN_FROM_HERE, compiler_to_run_from, compiler_name) compiler_to_run_from = compiler_name compiler_config.setdefault('input_step', self.INPUT_STEP_NAME) compilers[self.INPUT_STEP_NAME] = { 'class': 'daisychain.steps.input.InMemoryInput', 'output': config } instantiator = Instantiator(name=self.name, config=compilers) instantiator.run() executor = Executor(name=self.name, dependencies=list(instantiator.steps.values())) executor.execute() if compiler_to_run_from: return instantiator.steps[compiler_to_run_from].output else: return config
def test_execute_dry_run(): dep = test_step.MockStep(name='dep') assert not dep.checked_ready assert dep.status.pending e = Executor(dry_run=True, dependencies=[dep]) e.execute() assert dep.status.validated assert not dep.ran_once
def test_execute_dry_run(): dep = test_step.MockStep(name='dep') assert not dep.checked_ready assert dep.status.pending e = Executor(dry_run=True,dependencies=[dep]) e.execute() assert dep.status.validated assert not dep.ran_once
def compile(self, config): parenting_steps = dict() executor = Executor(name=self.name) for step_name, step_config in config[STEPS_KEY].items(): parenting_steps[step_name] = StepConfigInheritance.ParentedConfigurationStep(step_config, name=step_name) executor.dependencies.update(set(parenting_steps.values())) for parenting_step in list(parenting_steps.values()): parenting_step.evaluate_parentage(parenting_steps) executor.execute() return config
def test_execute_skip_failures(): dep = test_step.MockStep(name='mock_sibling_step', run_exception=RuntimeError("test_run_exception")) dep2 = test_step.MockStep(name='mock_sibling_step2', run_exception=RuntimeError("test_run_exception")) successful_dep = test_step.MockStep(name='successful_dep') parent = test_step.MockStep(name='mock_parent_step', dependencies=[dep, dep2, successful_dep]) assert parent.dependencies == {dep, dep2, successful_dep} successful_parent = test_step.MockStep(name='mock_successful_parent', dependencies=[successful_dep]) e = Executor(name='test_executor', on_failure=Executor.SKIP, dependencies=[parent, successful_parent]) e.execute() assert dep.failed assert dep2.failed assert successful_dep.finished assert parent.validated assert successful_parent.finished assert not e.execution.aborted
def test_cannot_find_command(): with patch('subprocess.Popen') as MockPopen: instance = MockPopen.returnvalue instance.returncode = 1 c = RunCommand(command='ls -l', poll_interval_seconds=0) executor = Executor(dependencies=[c]) try: executor.execute() except RuntimeError: pass except Exception as e: assert False, "Exception was supposed to be a RuntimeError but was {!r} instead".format(e) else: assert False, "Exception was not raised".format(e)
def test_execute_graceful_shutdown(): dep = test_step.MockStep(name='mock_sibling_step', run_exception=RuntimeError("test_run_exception")) dep2 = test_step.MockStep(name='mock_sibling_step2') dep2.run = lambda: dep2.status.set_validated() successful_dep = test_step.MockStep(name='successful_dep') parent = test_step.MockStep(name='mock_parent_step', dependencies=[dep, dep2, successful_dep]) assert parent.dependencies == {dep, dep2, successful_dep} successful_parent = test_step.MockStep(name='mock_successful_parent', dependencies=[successful_dep]) e = Executor(name='test_executor', on_failure=Executor.GRACEFUL_SHUTDOWN, dependencies=[parent, successful_parent]) e.execute() assert dep.status.failed assert dep2.status.finished or dep2.status.validated assert successful_dep.status.finished or successful_dep.status.validated assert parent.validated assert successful_parent.validated assert e.execution.aborted
def test_execute_graceful_shutdown_with_already_aborted_execution(): dep = test_step.MockStep(name='mock_sibling_step') dep2 = test_step.MockStep(name='mock_sibling_step2', run_exception=RuntimeError("test_run_exception")) successful_dep = test_step.MockStep(name='successful_dep') parent = test_step.MockStep(name='mock_parent_step', dependencies=[dep, dep2, successful_dep]) assert parent.dependencies == {dep, dep2, successful_dep} successful_parent = test_step.MockStep(name='mock_successful_parent', dependencies=[successful_dep]) e = Executor(name='test_executor', on_failure=Executor.GRACEFUL_SHUTDOWN, dependencies=[parent, successful_parent]) e.execution = Execution(executor=e) e.execution.aborted = True e.execute() assert dep.status.pending assert dep2.status.pending assert successful_dep.status.pending assert parent.status.pending assert successful_parent.status.pending
def test_prompting_during_execution(): with patch(input_function) as mock_raw_input: dep = test_step.MockStep(name='mock_step', run_exception=RuntimeError('Exception while running step')) e = Executor(name='test_executor', on_failure=Executor.PROMPT, dependencies=[dep]) global times_called times_called = 0 def raw_input_output(*args, **kwargs): global times_called times_called += 1 if times_called == 1: return 'r' elif times_called == 2: return 'f' assert times_called <= 2, "Called raw_input too many times" mock_raw_input.side_effect = raw_input_output e.execute() assert dep.status.finished assert times_called == 2
def test_failing_command(): with patch('subprocess.Popen') as MockPopen: validate_mock, command_mock = MagicMock(spec=RealPopen)(), MagicMock(spec=RealPopen)() MockPopen.side_effect = [validate_mock, command_mock] validate_mock.returncode = 0 command_mock.poll.return_value = 1 stdin = command_mock.stdin.return_value stdin.write.return_value = None stdin.close.return_value = None c = RunCommand(command='ls -l', poll_interval_seconds=0) executor = Executor(dependencies=[c]) try: executor.execute() except subprocess.CalledProcessError: pass except Exception as e: assert False, "Exception was supposed to be a CalledProcessError but was {!r} instead".format(e) else: assert False, "Exception was not raised".format(e)
def test_execute_graceful_shutdown(): dep = test_step.MockStep(name='mock_sibling_step', run_exception=RuntimeError("test_run_exception")) dep2 = test_step.MockStep(name='mock_sibling_step2') dep2.run = lambda: dep2.status.set_validated() successful_dep = test_step.MockStep(name='successful_dep') parent = test_step.MockStep(name='mock_parent_step', dependencies=[dep, dep2, successful_dep]) assert parent.dependencies == {dep, dep2, successful_dep} successful_parent = test_step.MockStep(name='mock_successful_parent', dependencies=[successful_dep]) e = Executor(name='test_executor', on_failure=Executor.GRACEFUL_SHUTDOWN, dependencies=[parent, successful_parent]) e.execute() assert dep.status.failed assert dep2.status.finished or dep2.status.validated assert successful_dep.status.finished or successful_dep.status.validated assert parent.validated assert successful_parent.validated assert e.execution.aborted
def test_execute_skip_failures(): dep = test_step.MockStep(name='mock_sibling_step', run_exception=RuntimeError("test_run_exception")) dep2 = test_step.MockStep(name='mock_sibling_step2', run_exception=RuntimeError("test_run_exception")) successful_dep = test_step.MockStep(name='successful_dep') parent = test_step.MockStep(name='mock_parent_step', dependencies=[dep, dep2, successful_dep]) assert parent.dependencies == {dep, dep2, successful_dep} successful_parent = test_step.MockStep(name='mock_successful_parent', dependencies=[successful_dep]) e = Executor(name='test_executor', on_failure=Executor.SKIP, dependencies=[parent, successful_parent]) e.execute() assert dep.failed assert dep2.failed assert successful_dep.finished assert parent.validated assert successful_parent.finished assert not e.execution.aborted
def test_execute_graceful_shutdown_with_already_aborted_execution(): dep = test_step.MockStep(name='mock_sibling_step') dep2 = test_step.MockStep(name='mock_sibling_step2', run_exception=RuntimeError("test_run_exception")) successful_dep = test_step.MockStep(name='successful_dep') parent = test_step.MockStep(name='mock_parent_step', dependencies=[dep, dep2, successful_dep]) assert parent.dependencies == {dep, dep2, successful_dep} successful_parent = test_step.MockStep(name='mock_successful_parent', dependencies=[successful_dep]) e = Executor(name='test_executor', on_failure=Executor.GRACEFUL_SHUTDOWN, dependencies=[parent, successful_parent]) e.execution = Execution(executor=e) e.execution.aborted = True e.execute() assert dep.status.pending assert dep2.status.pending assert successful_dep.status.pending assert parent.status.pending assert successful_parent.status.pending
def test_execute_check_status_failure_in_step(): dep = test_step.MockStep(name='mock_step', check_status_exception=TypeError("Exception while checking status")) e = Executor(name='test_executor', dependencies=[dep]) try: e.execute() except CheckStatusException: assert dep.failed else: assert False, "Should have thrown a CheckStatusException on failure" dep = test_step.MockStep(name='mock_step', check_status_exception=TypeError("Exception while checking status")) dep2 = test_step.MockStep(name='mock_failing_step_parent', dependencies=[dep]) e = Executor(name='test_executor', dependencies=[dep2]) try: e.execute() except CheckStatusException: assert dep2.validated assert dep.failed else: assert False, "Should have thrown a CheckStatusException on failure" dep = test_step.MockStep(name='mock_step') def raise_error(): raise RuntimeError("Exception while forwarding callback") dep.status.check = raise_error dep2 = test_step.MockStep(name='mock_failing_step_parent', dependencies=[dep]) e = Executor(name='test_executor', dependencies=[dep2]) try: e.execute() except CheckStatusException: assert dep2.validated assert dep.failed else: assert False, "Should have thrown a CheckStatusException on failure"
def run(self): instantiation_steps = self.get_instantiation_steps() self.log('instantiator').debug("Linking reference dependencies") for step in list(instantiation_steps.values()): step.evaluate_reference_dependencies(instantiation_steps) self.log('instantiator').debug("Running instantiator Tree") instantiation_executor = Executor(name=self.name) if self.root_log_id: instantiation_executor.root_log_id = self.root_log_id + '.' + instantiation_executor.root_log_id instantiation_executor.dependencies.update(set(instantiation_steps.values())) self.executor_output = instantiation_executor.execute() self.status.set_finished()
def run(self): instantiation_steps = self.get_instantiation_steps() self.log('instantiator').debug("Linking reference dependencies") for step in list(instantiation_steps.values()): step.evaluate_reference_dependencies(instantiation_steps) self.log('instantiator').debug("Running instantiator Tree") instantiation_executor = Executor(name=self.name) if self.root_log_id: instantiation_executor.root_log_id = self.root_log_id + '.' + instantiation_executor.root_log_id instantiation_executor.dependencies.update( set(instantiation_steps.values())) self.executor_output = instantiation_executor.execute() self.status.set_finished()
def test_auth_command(): with patch('subprocess.Popen') as MockPopen: validate_mock, auth_validate_mock, command_mock = MagicMock(spec=RealPopen)(), MagicMock(spec=RealPopen)(), MagicMock(spec=RealPopen)() MockPopen.side_effect = [validate_mock, auth_validate_mock, command_mock] validate_mock.returncode = 0 validate_mock.communicate.return_value = (None, None) auth_validate_mock.returncode = 0 auth_validate_mock.communicate.return_value = (None, None) command_mock.poll.return_value = 0 stdin = command_mock.stdin.return_value stdin.write.return_value = None stdin.close.return_value = None auth = BasicAuth(username='******', password='******') c = RunCommand(command='sudo -k mount -a', authentication=auth, poll_interval_seconds=0) executor = Executor(dependencies=[c]) executor.execute() assert validate_mock.communicate.call_count == 1 auth_validate_mock.communicate.assert_called_once_with('mockpassword\n') assert command_mock.poll.call_count == 1 command_mock.stdin.write.assert_called_once_with('mockpassword\n') assert command_mock.stdin.close.call_count == 1
def test_failing_auth_command(): with patch('subprocess.Popen') as MockPopen: validate_mock, auth_validate_mock = MagicMock(spec=RealPopen)(), MagicMock(spec=RealPopen)() MockPopen.side_effect = [validate_mock, auth_validate_mock] validate_mock.returncode = 0 validate_mock.communicate.return_value = (None, None) auth_validate_mock.returncode = 1 auth_validate_mock.communicate.return_value = (None, None) auth = BasicAuth(username='******', password='******') c = RunCommand(command='sudo -k mount -a', authentication=auth, poll_interval_seconds=0) executor = Executor(dependencies=[c]) try: executor.execute() except RuntimeError: pass except Exception as e: assert False, "Exception was supposed to be a RuntimeError but was {!r} instead".format(e) else: assert False, "Exception was not raised".format(e) assert validate_mock.communicate.call_count == 1 auth_validate_mock.communicate.assert_called_once_with('mockpassword\n')
def test_prompting_during_execution(): with patch(input_function) as mock_raw_input: dep = test_step.MockStep( name='mock_step', run_exception=RuntimeError('Exception while running step')) e = Executor(name='test_executor', on_failure=Executor.PROMPT, dependencies=[dep]) global times_called times_called = 0 def raw_input_output(*args, **kwargs): global times_called times_called += 1 if times_called == 1: return 'r' elif times_called == 2: return 'f' assert times_called <= 2, "Called raw_input too many times" mock_raw_input.side_effect = raw_input_output e.execute() assert dep.status.finished assert times_called == 2
def test_execute(): dep = test_step.MockStep(name='mock_step') e = Executor(name='test_executor', dependencies=[dep]) e.execute() assert dep.finished dep_named = test_step.MockStep(name='mock_step_named') dep = test_step.MockStep(name='mock_step', named_reference=dep_named) assert dep.named_reference is dep_named e = Executor(name='test_executor', dependencies=[dep]) e.execute() assert dep_named.finished assert dep.finished dep_dep = test_step.MockStep(name='mock_step_dep') dep = test_step.MockStep(name='mock_step', dependencies=[dep_dep]) assert dep.dependencies == {dep_dep} e = Executor(name='test_executor', dependencies=[dep]) assert e.dependencies == {dep} e.execute() assert dep_named.finished assert dep.finished dep = test_step.MockStep( name='mock_step', run_exception=RuntimeError('Exception while running step')) e = Executor(name='test_executor', dependencies=[dep]) try: e.execute() except RuntimeError: assert dep.failed else: assert False, "Should have thrown the error the step raised"
def test_execute(): dep = test_step.MockStep(name='mock_step') e = Executor(name='test_executor', dependencies=[dep]) e.execute() assert dep.finished dep_named = test_step.MockStep(name='mock_step_named') dep = test_step.MockStep(name='mock_step', named_reference=dep_named) assert dep.named_reference is dep_named e = Executor(name='test_executor', dependencies=[dep]) e.execute() assert dep_named.finished assert dep.finished dep_dep = test_step.MockStep(name='mock_step_dep') dep = test_step.MockStep(name='mock_step', dependencies=[dep_dep]) assert dep.dependencies == {dep_dep} e = Executor(name='test_executor', dependencies=[dep]) assert e.dependencies == {dep} e.execute() assert dep_named.finished assert dep.finished dep = test_step.MockStep(name='mock_step', run_exception=RuntimeError('Exception while running step')) e = Executor(name='test_executor', dependencies=[dep]) try: e.execute() except RuntimeError: assert dep.failed else: assert False, "Should have thrown the error the step raised"
def test_execute_check_status_failure_in_step(): dep = test_step.MockStep( name='mock_step', check_status_exception=TypeError("Exception while checking status")) e = Executor(name='test_executor', dependencies=[dep]) try: e.execute() except CheckStatusException: assert dep.failed else: assert False, "Should have thrown a CheckStatusException on failure" dep = test_step.MockStep( name='mock_step', check_status_exception=TypeError("Exception while checking status")) dep2 = test_step.MockStep(name='mock_failing_step_parent', dependencies=[dep]) e = Executor(name='test_executor', dependencies=[dep2]) try: e.execute() except CheckStatusException: assert dep2.validated assert dep.failed else: assert False, "Should have thrown a CheckStatusException on failure" dep = test_step.MockStep(name='mock_step') def raise_error(): raise RuntimeError("Exception while forwarding callback") dep.status.check = raise_error dep2 = test_step.MockStep(name='mock_failing_step_parent', dependencies=[dep]) e = Executor(name='test_executor', dependencies=[dep2]) try: e.execute() except CheckStatusException: assert dep2.validated assert dep.failed else: assert False, "Should have thrown a CheckStatusException on failure"
def test_basic_command(): c = RunCommand(command='ls -l', poll_interval_seconds=0) assert 'ls -l' in c.instructions executor = Executor(dependencies=[c]) executor.execute()