def compile(self, config):
if COMPILERS_KEY in config:
compilers = config.pop(COMPILERS_KEY)
if isinstance(compilers, list):
compilers = self._make_linear_chain(compilers, config)
elif not isinstance(compilers, dict):
raise TypeError('{!r} key must specify either a list or a dict of compilers'.format(COMPILERS_KEY))
compiler_to_run_from = None
for compiler_name, compiler_config in compilers.items():
if compiler_config.get(self.RUN_FROM_HERE, False):
assert compiler_to_run_from is None, "Only one compiler can specify '{!s}', but both {!r} and {!r} specify it".format(self.RUN_FROM_HERE, compiler_to_run_from, compiler_name)
compiler_to_run_from = compiler_name
compiler_config.setdefault('input_step', self.INPUT_STEP_NAME)
compilers[self.INPUT_STEP_NAME] = {'class': 'daisychain.steps.input.InMemoryInput', 'output': config}
instantiator = Instantiator(name=self.name, config=compilers)
instantiator.run()
executor = Executor(name=self.name, dependencies=list(instantiator.steps.values()))
executor.execute()
if compiler_to_run_from:
return instantiator.steps[compiler_to_run_from].output
else:
return config
def test_run():
w = ThreadedWait(seconds=0.5)
w2 = ThreadedWait(seconds=0.5)
executor = Executor(dependencies=[w, w2], scan_interval=0.01)
start_time = time.time()
executor.execute()
assert 0.5 < time.time() - start_time < 1.0
def test_manual_step():
m = Manual(instructions='Do something')
e = Executor(name='test_manual', dependencies=[m])
with patch(input_function) as mock_raw_input:
mock_raw_input.side_effect = ['a']
e.execute()
assert m.failed
def test_run():
w = Wait(seconds=0.1)
executor = Executor(dependencies=[w], scan_interval=0.01)
start_time = time.time()
executor.execute()
assert time.time() - start_time > 0.1
def test_run():
w = ThreadedWait(seconds=0.5)
w2 = ThreadedWait(seconds=0.5)
executor = Executor(dependencies=[w, w2], scan_interval=0.01)
start_time = time.time()
executor.execute()
assert 0.5 < time.time() - start_time < 1.0
def test_run():
w = Wait(seconds=0.1)
executor = Executor(dependencies=[w], scan_interval=0.01)
start_time = time.time()
executor.execute()
assert time.time() - start_time > 0.1
def compile(self, config):
if COMPILERS_KEY in config:
compilers = config.pop(COMPILERS_KEY)
if isinstance(compilers, list):
compilers = self._make_linear_chain(compilers, config)
elif not isinstance(compilers, dict):
raise TypeError(
'{!r} key must specify either a list or a dict of compilers'
.format(COMPILERS_KEY))
compiler_to_run_from = None
for compiler_name, compiler_config in compilers.items():
if compiler_config.get(self.RUN_FROM_HERE, False):
assert compiler_to_run_from is None, "Only one compiler can specify '{!s}', but both {!r} and {!r} specify it".format(
self.RUN_FROM_HERE, compiler_to_run_from,
compiler_name)
compiler_to_run_from = compiler_name
compiler_config.setdefault('input_step', self.INPUT_STEP_NAME)
compilers[self.INPUT_STEP_NAME] = {
'class': 'daisychain.steps.input.InMemoryInput',
'output': config
}
instantiator = Instantiator(name=self.name, config=compilers)
instantiator.run()
executor = Executor(name=self.name,
dependencies=list(instantiator.steps.values()))
executor.execute()
if compiler_to_run_from:
return instantiator.steps[compiler_to_run_from].output
else:
return config
def test_execute_dry_run():
dep = test_step.MockStep(name='dep')
assert not dep.checked_ready
assert dep.status.pending
e = Executor(dry_run=True, dependencies=[dep])
e.execute()
assert dep.status.validated
assert not dep.ran_once
def test_execute_dry_run():
dep = test_step.MockStep(name='dep')
assert not dep.checked_ready
assert dep.status.pending
e = Executor(dry_run=True,dependencies=[dep])
e.execute()
assert dep.status.validated
assert not dep.ran_once
def compile(self, config):
parenting_steps = dict()
executor = Executor(name=self.name)
for step_name, step_config in config[STEPS_KEY].items():
parenting_steps[step_name] = StepConfigInheritance.ParentedConfigurationStep(step_config, name=step_name)
executor.dependencies.update(set(parenting_steps.values()))
for parenting_step in list(parenting_steps.values()):
parenting_step.evaluate_parentage(parenting_steps)
executor.execute()
return config
def test_execute_skip_failures():
dep = test_step.MockStep(name='mock_sibling_step', run_exception=RuntimeError("test_run_exception"))
dep2 = test_step.MockStep(name='mock_sibling_step2', run_exception=RuntimeError("test_run_exception"))
successful_dep = test_step.MockStep(name='successful_dep')
parent = test_step.MockStep(name='mock_parent_step', dependencies=[dep, dep2, successful_dep])
assert parent.dependencies == {dep, dep2, successful_dep}
successful_parent = test_step.MockStep(name='mock_successful_parent', dependencies=[successful_dep])
e = Executor(name='test_executor', on_failure=Executor.SKIP, dependencies=[parent, successful_parent])
e.execute()
assert dep.failed
assert dep2.failed
assert successful_dep.finished
assert parent.validated
assert successful_parent.finished
assert not e.execution.aborted
def test_cannot_find_command():
with patch('subprocess.Popen') as MockPopen:
instance = MockPopen.returnvalue
instance.returncode = 1
c = RunCommand(command='ls -l', poll_interval_seconds=0)
executor = Executor(dependencies=[c])
try:
executor.execute()
except RuntimeError:
pass
except Exception as e:
assert False, "Exception was supposed to be a RuntimeError but was {!r} instead".format(e)
else:
assert False, "Exception was not raised".format(e)
def test_execute_graceful_shutdown():
dep = test_step.MockStep(name='mock_sibling_step', run_exception=RuntimeError("test_run_exception"))
dep2 = test_step.MockStep(name='mock_sibling_step2')
dep2.run = lambda: dep2.status.set_validated()
successful_dep = test_step.MockStep(name='successful_dep')
parent = test_step.MockStep(name='mock_parent_step', dependencies=[dep, dep2, successful_dep])
assert parent.dependencies == {dep, dep2, successful_dep}
successful_parent = test_step.MockStep(name='mock_successful_parent', dependencies=[successful_dep])
e = Executor(name='test_executor', on_failure=Executor.GRACEFUL_SHUTDOWN, dependencies=[parent, successful_parent])
e.execute()
assert dep.status.failed
assert dep2.status.finished or dep2.status.validated
assert successful_dep.status.finished or successful_dep.status.validated
assert parent.validated
assert successful_parent.validated
assert e.execution.aborted
def test_execute_graceful_shutdown_with_already_aborted_execution():
dep = test_step.MockStep(name='mock_sibling_step')
dep2 = test_step.MockStep(name='mock_sibling_step2', run_exception=RuntimeError("test_run_exception"))
successful_dep = test_step.MockStep(name='successful_dep')
parent = test_step.MockStep(name='mock_parent_step', dependencies=[dep, dep2, successful_dep])
assert parent.dependencies == {dep, dep2, successful_dep}
successful_parent = test_step.MockStep(name='mock_successful_parent', dependencies=[successful_dep])
e = Executor(name='test_executor', on_failure=Executor.GRACEFUL_SHUTDOWN, dependencies=[parent, successful_parent])
e.execution = Execution(executor=e)
e.execution.aborted = True
e.execute()
assert dep.status.pending
assert dep2.status.pending
assert successful_dep.status.pending
assert parent.status.pending
assert successful_parent.status.pending
def test_prompting_during_execution():
with patch(input_function) as mock_raw_input:
dep = test_step.MockStep(name='mock_step', run_exception=RuntimeError('Exception while running step'))
e = Executor(name='test_executor', on_failure=Executor.PROMPT, dependencies=[dep])
global times_called
times_called = 0
def raw_input_output(*args, **kwargs):
global times_called
times_called += 1
if times_called == 1:
return 'r'
elif times_called == 2:
return 'f'
assert times_called <= 2, "Called raw_input too many times"
mock_raw_input.side_effect = raw_input_output
e.execute()
assert dep.status.finished
assert times_called == 2
def test_failing_command():
with patch('subprocess.Popen') as MockPopen:
validate_mock, command_mock = MagicMock(spec=RealPopen)(), MagicMock(spec=RealPopen)()
MockPopen.side_effect = [validate_mock, command_mock]
validate_mock.returncode = 0
command_mock.poll.return_value = 1
stdin = command_mock.stdin.return_value
stdin.write.return_value = None
stdin.close.return_value = None
c = RunCommand(command='ls -l', poll_interval_seconds=0)
executor = Executor(dependencies=[c])
try:
executor.execute()
except subprocess.CalledProcessError:
pass
except Exception as e:
assert False, "Exception was supposed to be a CalledProcessError but was {!r} instead".format(e)
else:
assert False, "Exception was not raised".format(e)
def test_execute_graceful_shutdown():
dep = test_step.MockStep(name='mock_sibling_step',
run_exception=RuntimeError("test_run_exception"))
dep2 = test_step.MockStep(name='mock_sibling_step2')
dep2.run = lambda: dep2.status.set_validated()
successful_dep = test_step.MockStep(name='successful_dep')
parent = test_step.MockStep(name='mock_parent_step',
dependencies=[dep, dep2, successful_dep])
assert parent.dependencies == {dep, dep2, successful_dep}
successful_parent = test_step.MockStep(name='mock_successful_parent',
dependencies=[successful_dep])
e = Executor(name='test_executor',
on_failure=Executor.GRACEFUL_SHUTDOWN,
dependencies=[parent, successful_parent])
e.execute()
assert dep.status.failed
assert dep2.status.finished or dep2.status.validated
assert successful_dep.status.finished or successful_dep.status.validated
assert parent.validated
assert successful_parent.validated
assert e.execution.aborted
def test_execute_skip_failures():
dep = test_step.MockStep(name='mock_sibling_step',
run_exception=RuntimeError("test_run_exception"))
dep2 = test_step.MockStep(name='mock_sibling_step2',
run_exception=RuntimeError("test_run_exception"))
successful_dep = test_step.MockStep(name='successful_dep')
parent = test_step.MockStep(name='mock_parent_step',
dependencies=[dep, dep2, successful_dep])
assert parent.dependencies == {dep, dep2, successful_dep}
successful_parent = test_step.MockStep(name='mock_successful_parent',
dependencies=[successful_dep])
e = Executor(name='test_executor',
on_failure=Executor.SKIP,
dependencies=[parent, successful_parent])
e.execute()
assert dep.failed
assert dep2.failed
assert successful_dep.finished
assert parent.validated
assert successful_parent.finished
assert not e.execution.aborted
def test_execute_graceful_shutdown_with_already_aborted_execution():
dep = test_step.MockStep(name='mock_sibling_step')
dep2 = test_step.MockStep(name='mock_sibling_step2',
run_exception=RuntimeError("test_run_exception"))
successful_dep = test_step.MockStep(name='successful_dep')
parent = test_step.MockStep(name='mock_parent_step',
dependencies=[dep, dep2, successful_dep])
assert parent.dependencies == {dep, dep2, successful_dep}
successful_parent = test_step.MockStep(name='mock_successful_parent',
dependencies=[successful_dep])
e = Executor(name='test_executor',
on_failure=Executor.GRACEFUL_SHUTDOWN,
dependencies=[parent, successful_parent])
e.execution = Execution(executor=e)
e.execution.aborted = True
e.execute()
assert dep.status.pending
assert dep2.status.pending
assert successful_dep.status.pending
assert parent.status.pending
assert successful_parent.status.pending
def test_execute_check_status_failure_in_step():
dep = test_step.MockStep(name='mock_step', check_status_exception=TypeError("Exception while checking status"))
e = Executor(name='test_executor', dependencies=[dep])
try:
e.execute()
except CheckStatusException:
assert dep.failed
else:
assert False, "Should have thrown a CheckStatusException on failure"
dep = test_step.MockStep(name='mock_step', check_status_exception=TypeError("Exception while checking status"))
dep2 = test_step.MockStep(name='mock_failing_step_parent', dependencies=[dep])
e = Executor(name='test_executor', dependencies=[dep2])
try:
e.execute()
except CheckStatusException:
assert dep2.validated
assert dep.failed
else:
assert False, "Should have thrown a CheckStatusException on failure"
dep = test_step.MockStep(name='mock_step')
def raise_error():
raise RuntimeError("Exception while forwarding callback")
dep.status.check = raise_error
dep2 = test_step.MockStep(name='mock_failing_step_parent', dependencies=[dep])
e = Executor(name='test_executor', dependencies=[dep2])
try:
e.execute()
except CheckStatusException:
assert dep2.validated
assert dep.failed
else:
assert False, "Should have thrown a CheckStatusException on failure"
def run(self):
instantiation_steps = self.get_instantiation_steps()
self.log('instantiator').debug("Linking reference dependencies")
for step in list(instantiation_steps.values()):
step.evaluate_reference_dependencies(instantiation_steps)
self.log('instantiator').debug("Running instantiator Tree")
instantiation_executor = Executor(name=self.name)
if self.root_log_id:
instantiation_executor.root_log_id = self.root_log_id + '.' + instantiation_executor.root_log_id
instantiation_executor.dependencies.update(set(instantiation_steps.values()))
self.executor_output = instantiation_executor.execute()
self.status.set_finished()
def run(self):
instantiation_steps = self.get_instantiation_steps()
self.log('instantiator').debug("Linking reference dependencies")
for step in list(instantiation_steps.values()):
step.evaluate_reference_dependencies(instantiation_steps)
self.log('instantiator').debug("Running instantiator Tree")
instantiation_executor = Executor(name=self.name)
if self.root_log_id:
instantiation_executor.root_log_id = self.root_log_id + '.' + instantiation_executor.root_log_id
instantiation_executor.dependencies.update(
set(instantiation_steps.values()))
self.executor_output = instantiation_executor.execute()
self.status.set_finished()
def test_auth_command():
with patch('subprocess.Popen') as MockPopen:
validate_mock, auth_validate_mock, command_mock = MagicMock(spec=RealPopen)(), MagicMock(spec=RealPopen)(), MagicMock(spec=RealPopen)()
MockPopen.side_effect = [validate_mock, auth_validate_mock, command_mock]
validate_mock.returncode = 0
validate_mock.communicate.return_value = (None, None)
auth_validate_mock.returncode = 0
auth_validate_mock.communicate.return_value = (None, None)
command_mock.poll.return_value = 0
stdin = command_mock.stdin.return_value
stdin.write.return_value = None
stdin.close.return_value = None
auth = BasicAuth(username='******', password='******')
c = RunCommand(command='sudo -k mount -a', authentication=auth, poll_interval_seconds=0)
executor = Executor(dependencies=[c])
executor.execute()
assert validate_mock.communicate.call_count == 1
auth_validate_mock.communicate.assert_called_once_with('mockpassword\n')
assert command_mock.poll.call_count == 1
command_mock.stdin.write.assert_called_once_with('mockpassword\n')
assert command_mock.stdin.close.call_count == 1
def test_failing_auth_command():
with patch('subprocess.Popen') as MockPopen:
validate_mock, auth_validate_mock = MagicMock(spec=RealPopen)(), MagicMock(spec=RealPopen)()
MockPopen.side_effect = [validate_mock, auth_validate_mock]
validate_mock.returncode = 0
validate_mock.communicate.return_value = (None, None)
auth_validate_mock.returncode = 1
auth_validate_mock.communicate.return_value = (None, None)
auth = BasicAuth(username='******', password='******')
c = RunCommand(command='sudo -k mount -a', authentication=auth, poll_interval_seconds=0)
executor = Executor(dependencies=[c])
try:
executor.execute()
except RuntimeError:
pass
except Exception as e:
assert False, "Exception was supposed to be a RuntimeError but was {!r} instead".format(e)
else:
assert False, "Exception was not raised".format(e)
assert validate_mock.communicate.call_count == 1
auth_validate_mock.communicate.assert_called_once_with('mockpassword\n')
def test_prompting_during_execution():
with patch(input_function) as mock_raw_input:
dep = test_step.MockStep(
name='mock_step',
run_exception=RuntimeError('Exception while running step'))
e = Executor(name='test_executor',
on_failure=Executor.PROMPT,
dependencies=[dep])
global times_called
times_called = 0
def raw_input_output(*args, **kwargs):
global times_called
times_called += 1
if times_called == 1:
return 'r'
elif times_called == 2:
return 'f'
assert times_called <= 2, "Called raw_input too many times"
mock_raw_input.side_effect = raw_input_output
e.execute()
assert dep.status.finished
assert times_called == 2
def test_execute():
dep = test_step.MockStep(name='mock_step')
e = Executor(name='test_executor', dependencies=[dep])
e.execute()
assert dep.finished
dep_named = test_step.MockStep(name='mock_step_named')
dep = test_step.MockStep(name='mock_step', named_reference=dep_named)
assert dep.named_reference is dep_named
e = Executor(name='test_executor', dependencies=[dep])
e.execute()
assert dep_named.finished
assert dep.finished
dep_dep = test_step.MockStep(name='mock_step_dep')
dep = test_step.MockStep(name='mock_step', dependencies=[dep_dep])
assert dep.dependencies == {dep_dep}
e = Executor(name='test_executor', dependencies=[dep])
assert e.dependencies == {dep}
e.execute()
assert dep_named.finished
assert dep.finished
dep = test_step.MockStep(
name='mock_step',
run_exception=RuntimeError('Exception while running step'))
e = Executor(name='test_executor', dependencies=[dep])
try:
e.execute()
except RuntimeError:
assert dep.failed
else:
assert False, "Should have thrown the error the step raised"
def test_execute():
dep = test_step.MockStep(name='mock_step')
e = Executor(name='test_executor', dependencies=[dep])
e.execute()
assert dep.finished
dep_named = test_step.MockStep(name='mock_step_named')
dep = test_step.MockStep(name='mock_step', named_reference=dep_named)
assert dep.named_reference is dep_named
e = Executor(name='test_executor', dependencies=[dep])
e.execute()
assert dep_named.finished
assert dep.finished
dep_dep = test_step.MockStep(name='mock_step_dep')
dep = test_step.MockStep(name='mock_step', dependencies=[dep_dep])
assert dep.dependencies == {dep_dep}
e = Executor(name='test_executor', dependencies=[dep])
assert e.dependencies == {dep}
e.execute()
assert dep_named.finished
assert dep.finished
dep = test_step.MockStep(name='mock_step', run_exception=RuntimeError('Exception while running step'))
e = Executor(name='test_executor', dependencies=[dep])
try:
e.execute()
except RuntimeError:
assert dep.failed
else:
assert False, "Should have thrown the error the step raised"
def test_execute_check_status_failure_in_step():
dep = test_step.MockStep(
name='mock_step',
check_status_exception=TypeError("Exception while checking status"))
e = Executor(name='test_executor', dependencies=[dep])
try:
e.execute()
except CheckStatusException:
assert dep.failed
else:
assert False, "Should have thrown a CheckStatusException on failure"
dep = test_step.MockStep(
name='mock_step',
check_status_exception=TypeError("Exception while checking status"))
dep2 = test_step.MockStep(name='mock_failing_step_parent',
dependencies=[dep])
e = Executor(name='test_executor', dependencies=[dep2])
try:
e.execute()
except CheckStatusException:
assert dep2.validated
assert dep.failed
else:
assert False, "Should have thrown a CheckStatusException on failure"
dep = test_step.MockStep(name='mock_step')
def raise_error():
raise RuntimeError("Exception while forwarding callback")
dep.status.check = raise_error
dep2 = test_step.MockStep(name='mock_failing_step_parent',
dependencies=[dep])
e = Executor(name='test_executor', dependencies=[dep2])
try:
e.execute()
except CheckStatusException:
assert dep2.validated
assert dep.failed
else:
assert False, "Should have thrown a CheckStatusException on failure"
def test_basic_command():
c = RunCommand(command='ls -l', poll_interval_seconds=0)
assert 'ls -l' in c.instructions
executor = Executor(dependencies=[c])
executor.execute()