def test_executor_configure_invalid_steps():
from plumber.operators import Executor
executor = Executor()
try:
executor.configure({STEPS: {'step 1': 'step 1'}})
pytest.fail('Executor should not be configured without right steps')
except Exception as e:
assert type(e) is ConfigError
def test_executor_configure_no_steps():
from plumber.operators import Executor
executor = Executor()
try:
executor.configure({})
pytest.fail('Executor should not be configured without steps')
except Exception as e:
assert type(e) is ConfigError
def test_executor_configure_invalid_timeout():
CONFIG = {
STEPS: ['echo "Hello"', 'echo "World"'],
BATCH: False,
TIMEOUT: '100'
}
from plumber.operators import Executor
executor = Executor()
try:
executor.configure(CONFIG)
except Exception as e:
assert type(e) is ConfigError
def test_executor_configure():
CONFIG = {
STEPS: ['echo "Hello"', 'echo "World"'],
BATCH: False,
TIMEOUT: 100
}
from plumber.operators import Executor
executor = Executor()
executor.configure(CONFIG)
assert executor.steps[0] == CONFIG[STEPS][0]
assert executor.steps[1] == CONFIG[STEPS][1]
assert executor.batch is False
assert executor.timeout == 100
assert executor.results is not None
def configure(self, config):
prehooks = get_or_default(config, PREHOOK, None, list)
if prehooks is not None:
self.prehooks = []
for prehook_config in prehooks:
executor = Executor()
executor.configure(prehook_config)
self.prehooks.append(executor)
posthooks = get_or_default(config, POSTHOOK, None, list)
if posthooks is not None:
self.posthooks = []
self.posthooks_success = []
self.posthooks_failure = []
for posthook_config in posthooks:
executor = Executor()
executor.configure(posthook_config)
condition = get_or_default(posthook_config, CONDITION, ALWAYS,
str).lower()
if condition == ALWAYS:
self.posthooks.append(executor)
elif condition == FAILURE:
self.posthooks_failure.append(executor)
elif condition == SUCCESS:
self.posthooks_success.append(executor)
else:
raise ConfigError(
'Invalid execution condition specified on prehook: {}'.
format(condition))
def configure(self, config, checkpoint):
super(PlumberPipe, self).configure(config=config)
id = get_or_default(config, ID, None, str)
if id is None:
raise ConfigError(
'Id not specified for pipe in configuration file:\n{}'.format(
yaml.dump(config)))
self.config = config
self.checkpoint = checkpoint
conditions = get_or_default(config, CONDITIONS, None, list)
if conditions is not None:
self.conditions = []
declared_conditions = set()
for condition_config in conditions:
id = get_or_default(condition_config, ID, None, str)
if id is None:
raise ConfigError(
'Id not specified for condition in the configuration file:\n{}'
.format(yaml.dump(condition_config)))
if id in declared_conditions:
raise ConfigError(
'Multiple conditions specified with the id: {}\n'.
format(id, yaml.dump(condition_config)))
declared_conditions.add(id)
self.conditions.append({
ID:
id,
CONDITION:
_create_conditional(
condition_config,
get_or_default(checkpoint, id, {}, dict))
})
actions = get_or_default(config, ACTIONS, None, dict)
if actions is not None:
self.actions = Executor()
self.actions.configure(actions)
def test_executor_execute_batch():
CONFIG = {
STEPS: ['echo "Hello"', 'echo "World"'],
BATCH: True,
}
from plumber.operators import Executor
executor = Executor()
executor.configure(CONFIG)
executor.execute()
assert len(executor.results) == 1
assert executor.results[0][RETURN_CODE] == 0
assert executor.results[0][STDOUT].decode(UTF8) == 'Hello\nWorld\n'
assert executor.results[0][STDERR].decode(UTF8) == ''
assert executor.results[0][STEP] == '\n echo "Hello"\n echo "World"'
def test_executor_execute_batch_error():
CONFIG = {
STEPS: ['echo "Hello"', 'echoi "World"'],
BATCH: True,
}
from plumber.operators import Executor
executor = Executor()
executor.configure(CONFIG)
try:
executor.execute()
except Exception as e:
assert type(e) is ExecutionFailure
assert len(executor.results) == 1
assert executor.results[0][RETURN_CODE] != 0
assert executor.results[0][STDOUT].decode(UTF8) == 'Hello\n'
assert executor.results[0][STDERR].decode(UTF8) != ''
assert executor.results[0][STEP] == '\n echo "Hello"\n echoi "World"'
def test_executor_execute_error():
CONFIG = {
STEPS: ['echi "Hello"', 'echo "World"'],
BATCH: False,
}
from plumber.operators import Executor
executor = Executor()
executor.configure(CONFIG)
try:
executor.execute()
pytest.fail('Executor should raise exception on invalid command')
except Exception as e:
assert type(e) is ExecutionFailure
assert len(executor.results) == 1
assert executor.results[0][RETURN_CODE] != 0
assert executor.results[0][STDOUT].decode(UTF8) == ''
assert executor.results[0][STDERR].decode(UTF8) != ''
assert executor.results[0][STEP] == 'echi "Hello"'
def test_executor_execute_error_timeout():
CONFIG = {
STEPS: ['sleep 2', 'echo "World"'],
TIMEOUT: 1,
BATCH: False,
}
from plumber.operators import Executor
executor = Executor()
executor.configure(CONFIG)
try:
executor.execute()
pytest.fail(
'Executor should raise timeout exception if timeout is set')
except Exception as e:
assert type(e) is ExecutionFailure
assert len(executor.results) == 1
assert executor.results[0][RETURN_CODE] == 130
assert executor.results[0][STDOUT] == None
assert executor.results[0][STDERR] != ''
assert executor.results[0][STEP] == 'sleep 2'
def test_executor_execute_get_results():
CONFIG = {
STEPS: ['echo "Hello"', 'echo "World"'],
BATCH: False,
}
from plumber.operators import Executor
executor = Executor()
executor.configure(CONFIG)
assert len(executor.get_results()) == 0
executor.execute()
assert len(executor.get_results()) == 2
assert executor.get_results()[0][RETURN_CODE] == 0
assert executor.get_results()[0][STDOUT].decode(UTF8) == 'Hello\n'
assert executor.get_results()[0][STDERR].decode(UTF8) == ''
assert executor.get_results()[0][STEP] == 'echo "Hello"'
assert executor.get_results()[1][RETURN_CODE] == 0
assert executor.get_results()[1][STDOUT].decode(UTF8) == 'World\n'
assert executor.get_results()[1][STDERR].decode(UTF8) == ''
assert executor.get_results()[1][STEP] == 'echo "World"'
class PlumberPipe(Hooked):
def __init__(self):
super(PlumberPipe, self).__init__()
self.config = None
self.conditions = None
self.actions = None
self.checkpoint = None
def configure(self, config, checkpoint):
super(PlumberPipe, self).configure(config=config)
id = get_or_default(config, ID, None, str)
if id is None:
raise ConfigError(
'Id not specified for pipe in configuration file:\n{}'.format(
yaml.dump(config)))
self.config = config
self.checkpoint = checkpoint
conditions = get_or_default(config, CONDITIONS, None, list)
if conditions is not None:
self.conditions = []
declared_conditions = set()
for condition_config in conditions:
id = get_or_default(condition_config, ID, None, str)
if id is None:
raise ConfigError(
'Id not specified for condition in the configuration file:\n{}'
.format(yaml.dump(condition_config)))
if id in declared_conditions:
raise ConfigError(
'Multiple conditions specified with the id: {}\n'.
format(id, yaml.dump(condition_config)))
declared_conditions.add(id)
self.conditions.append({
ID:
id,
CONDITION:
_create_conditional(
condition_config,
get_or_default(checkpoint, id, {}, dict))
})
actions = get_or_default(config, ACTIONS, None, dict)
if actions is not None:
self.actions = Executor()
self.actions.configure(actions)
def evaluate(self):
if self.conditions is None:
return True
expression = get_or_default(self.config, EXPRESSION, None, str)
if expression is not None:
exp_values = {}
for condition in self.conditions:
exp_values[condition[ID]] = condition[CONDITION].evaluate()
return evaluate_expression(expression, exp_values)
else:
for condition in self.conditions:
if condition[CONDITION].evaluate():
return True
return False
def execute(self):
self.actions.execute()
def get_new_checkpoint(self):
if self.conditions is None:
return None
for condition in self.conditions:
self.checkpoint[
condition[ID]] = condition[CONDITION].create_checkpoint()
return self.checkpoint
def run_prehooks(self):
if self.prehooks is not None:
LOG.log(
PLUMBER_LOGS,
wrap_in_dividers("Running prehooks for {}".format(
self.config[ID]),
divider_char='-'))
super(PlumberPipe, self).run_prehooks()
def run_posthooks(self, last_result):
if self.posthooks is not None or (
last_result == SUCCESS and self.posthooks_success
is not None) or (last_result == FAILURE
and self.posthooks_failure is not None):
LOG.log(
PLUMBER_LOGS,
wrap_in_dividers("Running posthooks for {}".format(
self.config[ID]),
divider_char='-'))
super(PlumberPipe, self).run_posthooks(last_result)