def test_fetchyaml_pass():
"""Relative path to yaml should succeed.
Strictly speaking not a unit test.
"""
context = Context({
'ok1': 'ov1',
'fetchYaml': {
'path': './tests/testfiles/dict.yaml'
}
})
filefetcher.run_step(context)
assert context, "context shouldn't be None"
assert len(context) == 7, "context should have 7 items"
assert context['ok1'] == 'ov1'
assert context['fetchYaml']['path'] == './tests/testfiles/dict.yaml'
assert context['key2'] == 'value2', "key2 should be value2"
assert len(context['key4']['k42']) == 3, "3 items in k42"
assert 'k42list2' in context['key4']['k42'], "k42 containts k42list2"
assert context['key4']['k43'], "k43 is True"
assert context['key4']['k44'] == 77, "k44 is 77"
assert len(context['key5']) == 2, "2 items in key5"
def test_pycode_replace_objects():
"""Rebind mutable and immutable objects in eval scope."""
pycode = """\
context['alist'] = [0, 1, 2]
context['adict'] = {'a': 'b', 'c': 'd'}
context['anint'] = 456
context['mutate_me'].append(12)
context['astring'] = 'updated'
"""
context = Context({'alist': [0, 1],
'adict': {'a': 'b'},
'anint': 123,
'mutate_me': [10, 11],
'astring': 'a string',
'pycode': pycode})
pypyr.steps.py.run_step(context)
assert context == {'alist': [0, 1, 2],
'adict': {'a': 'b', 'c': 'd'},
'anint': 456,
'mutate_me': [10, 11, 12],
'astring': 'updated',
'pycode': pycode}
def test_pype_get_arguments_group_str():
"""Parse group as str input from context."""
context = Context({'pype': {
'name': 'pipe name',
'groups': 'gr',
}})
(pipeline_name, args, out, use_parent_context, pipe_arg, skip_parse,
raise_error, loader, groups, success_group,
failure_group) = pype.get_arguments(context)
assert pipeline_name == 'pipe name'
assert not args
assert not out
assert use_parent_context
assert isinstance(use_parent_context, bool)
assert skip_parse
assert isinstance(skip_parse, bool)
assert raise_error
assert isinstance(raise_error, bool)
assert loader is None
assert groups == ['gr']
assert success_group is None
assert failure_group is None
def test_envget_pass_with_substitutions():
"""Env get success case with string substitutions."""
os.environ['ARB_DELETE_ME1'] = 'arb value from $ENV ARB_DELETE_ME1'
context = Context({
'key1':
'value1',
'key2':
'value2',
'env_val1':
'ARB_DELETE_ME1',
'env_val2':
'ARB_DELETE_ME2',
'default_val':
'blah',
'key_val':
'key3',
'envGet': [{
'env': '{env_val1}',
'key': '{key_val}',
'default': 'blah'
}, {
'env': '{env_val2}',
'key': 'key4',
'default': '{default_val}'
}]
})
pypyr.steps.envget.run_step(context)
del os.environ['ARB_DELETE_ME1']
assert context['key1'] == 'value1'
assert context['key2'] == 'value2'
assert context['key3'] == 'arb value from $ENV ARB_DELETE_ME1'
assert context['key4'] == 'blah'
def test_aws_wait_pass_no_args(mock_waiter):
"""Successful run with no client args"""
context = Context({
'k1': 'v1',
'awsWaitIn': {
'serviceName': 'service name',
'waiterName': 'waiter_name',
'arbKey': 'arb_value'
}})
wait.run_step(context)
assert len(context) == 2
assert context['k1'] == 'v1'
assert context['awsWaitIn'] == {
'serviceName': 'service name',
'waiterName': 'waiter_name',
'arbKey': 'arb_value'
}
mock_waiter.assert_called_once_with(service_name='service name',
waiter_name='waiter_name',
waiter_args=None,
wait_args=None,
)
def test_envget_pass():
"""envget success case"""
os.environ['ARB_DELETE_ME1'] = 'arb value from $ENV ARB_DELETE_ME1'
os.environ['ARB_DELETE_ME2'] = 'arb value from $ENV ARB_DELETE_ME2'
context = Context({
'key1': 'value1',
'key2': 'value2',
'key3': 'value3',
'envGet': {
'key2': 'ARB_DELETE_ME1',
'key4': 'ARB_DELETE_ME2'
}
})
pypyr.steps.env.env_get(context)
assert context['key1'] == 'value1'
assert context['key2'] == 'arb value from $ENV ARB_DELETE_ME1'
assert context['key3'] == 'value3'
assert context['key4'] == 'arb value from $ENV ARB_DELETE_ME2'
del os.environ['ARB_DELETE_ME1']
del os.environ['ARB_DELETE_ME2']
def test_pyimport():
"""Import namespace saved to context."""
source = """\
import math
import tests.arbpack.arbmod
import tests.arbpack.arbmod2
from tests.arbpack.arbmultiattr import arb_attr, arb_func as y
"""
context = Context({'pyImport': source})
pyimport.run_step(context)
ns = context._pystring_globals
len(ns) == 4
assert ns['math'].sqrt(4) == 2
# no return value but shouldn't raise not found.
ns['tests'].arbpack.arbmod.arbmod_attribute()
assert ns['arb_attr'] == 123.456
assert ns['y']('ab3') == 'ab3'
# parent did not import anything NOT specified.
# tests.arbpack.arbstep exists but wasn't specified for import.
assert not hasattr(ns['tests'].arbpack, 'arbstep')
def test_pype_get_args_no_parent_context():
"""If args set use_parent_context should default False."""
context = Context({'pype': {
'name': 'pipe name',
'args': {
'a': 'b'
},
}})
(pipeline_name, args, out, use_parent_context, pipe_arg, skip_parse,
raise_error, loader, groups, success_group,
failure_group) = pype.get_arguments(context)
assert pipeline_name == 'pipe name'
assert args == {'a': 'b'}
assert out is None
assert not use_parent_context
assert pipe_arg is None
assert skip_parse
assert raise_error
assert not loader
assert not groups
assert not success_group
assert not failure_group
def test_assert_raises_on_assertthis_not_equals_dict_to_dict_substitutions():
"""Assert this string does not equal assertEquals dict."""
context = Context({
'k1': 'v1',
'k2': 'v2',
'assert': {
'this': {
'k1': 1,
'k2': [2, '{k1}'],
'k3': False
},
'equals': {
'k1': 1,
'k2': [2, '{k2}'],
'k3': False
}
}
})
with pytest.raises(AssertionError) as err_info:
assert_step.run_step(context)
assert str(
err_info.value) == ("assert assert['this'] is of type dict and does "
"not equal assert['equals'] of type dict.")
def test_pathcheck_single(mock_glob):
"""Single path ok."""
context = Context({'ok1': 'ov1', 'pathCheck': './arb/x'})
mock_glob.return_value = ['./foundfile']
with patch_logger('pypyr.steps.pathcheck',
logging.INFO) as mock_logger_info:
pathchecker.run_step(context)
mock_logger_info.assert_called_once_with('checked 1 path(s) and found 1')
assert context, "context shouldn't be None"
assert len(context) == 3, "context should have 3 items"
assert context['ok1'] == 'ov1'
assert context['pathCheck'] == './arb/x'
assert context["pathCheckOut"] == {
'./arb/x': {
'exists': True,
'count': 1,
'found': ['./foundfile']
}
}
mock_glob.assert_called_once_with('./arb/x')
def test_pype_use_parent_context_with_swallow(mock_run_pipeline):
"""pype swallowing error in child pipeline."""
context = Context({
'pype': {
'name': 'pipe name',
'pipeArg': 'argument here',
'useParentContext': True,
'skipParse': True,
'raiseError': False
}
})
logger = logging.getLogger('pypyr.steps.pype')
with patch.object(logger, 'error') as mock_logger_error:
pype.run_step(context)
mock_run_pipeline.assert_called_once_with(
pipeline_name='pipe name',
pipeline_context_input='argument here',
context=context,
parse_input=False)
mock_logger_error.assert_called_once_with(
'Something went wrong pyping pipe name. RuntimeError: whoops')
def test_tar_only_calls_archive():
"""Only calls archive if only archive specified."""
context = Context({
'key1': 'value1',
'key2': 'value2',
'key3': 'value3',
'tar': {
'archive': [{
'in': 'key2',
'out': 'ARB_GET_ME1'
}, {
'in': 'key4',
'out': 'ARB_GET_ME2'
}]
}
})
with patch.multiple('pypyr.steps.tar',
tar_archive=DEFAULT,
tar_extract=DEFAULT) as mock_tar:
pypyr.steps.tar.run_step(context)
mock_tar['tar_extract'].assert_not_called()
mock_tar['tar_archive'].assert_called_once()
def test_waitfor_pass_all_args(mock_sleep, mock_service):
"""Successful run with client and method args pass on #2."""
mock_service.side_effect = [
{
'rk1': 'rv1',
'rk2': 'rv2'
}, # 1
{
'rk1': 'rv1',
'rk2': False
}, # 2
{
'rk1': 'rv1',
'rk2': 'rv2'
}, # 3
]
context = Context({
'k1': 'v1',
'awsWaitFor': {
'awsClientIn': {
'serviceName': 'service name',
'methodName': 'method_name',
'arbKey': 'arb_value',
'clientArgs': {
'ck1': 'cv1',
'ck2': 'cv2'
},
'methodArgs': {
'mk1': 'mv1',
'mk2': 'mv2'
}
},
'waitForField': '{rk2}',
'toBe': False,
'pollInterval': 99
}
})
waitfor_step.run_step(context)
assert len(context) == 3
assert context['k1'] == 'v1'
assert context['awsWaitFor']['awsClientIn'] == {
'serviceName': 'service name',
'methodName': 'method_name',
'arbKey': 'arb_value',
'clientArgs': {
'ck1': 'cv1',
'ck2': 'cv2'
},
'methodArgs': {
'mk1': 'mv1',
'mk2': 'mv2'
}
}
assert not context['awsWaitForTimedOut']
mock_sleep.assert_called_once_with(99)
mock_service.assert_called_with(
service_name='service name',
method_name='method_name',
client_args={
'ck1': 'cv1',
'ck2': 'cv2'
},
operation_args={
'mk1': 'mv1',
'mk2': 'mv2'
},
)
def test_waitfor_pass_client_args(mock_sleep, mock_service):
"""Successful run with client args pass on 3."""
mock_service.side_effect = [
{
'rk1': 'rv1',
'rk2': 'rv2'
}, # 1
{
'rk1': 'rv1',
'rk2': 'rv2'
}, # 2
{
'rk1': 'rv1',
'rk2': 'xxx'
}, # 3
{
'rk1': 'rv1',
'rk2': 'rv2'
} # 4
]
context = Context({
'k1': 'v1',
'awsWaitFor': {
'awsClientIn': {
'serviceName': 'service name',
'methodName': 'method_name',
'arbKey': 'arb_value',
'clientArgs': {
'ck1': 'cv1',
'ck2': 'cv2'
}
},
'waitForField': '{rk2}',
'toBe': 'xxx'
}
})
waitfor_step.run_step(context)
assert len(context) == 3
assert context['k1'] == 'v1'
assert context['awsWaitFor']['awsClientIn'] == {
'serviceName': 'service name',
'methodName': 'method_name',
'arbKey': 'arb_value',
'clientArgs': {
'ck1': 'cv1',
'ck2': 'cv2'
}
}
assert not context['awsWaitForTimedOut']
mock_service.assert_called_with(
service_name='service name',
method_name='method_name',
client_args={
'ck1': 'cv1',
'ck2': 'cv2'
},
operation_args=None,
)
mock_sleep.call_count == 3
mock_sleep.assert_called_with(30)
def run_step(context):
"""Run another pipeline from this step.
The parent pipeline is the current, executing pipeline. The invoked, or
child pipeline is the pipeline you are calling from this step.
Args:
context: dictionary-like pypyr.context.Context. context is mandatory.
Uses the following context keys in context:
- pype
- name. mandatory. str. Name of pipeline to execute. This
{name}.yaml must exist in the working directory/pipelines
dir.
- args. optional. dict. Create the context of the called
pipeline with these keys & values. If args specified,
will not pass the parent context unless you explicitly set
useParentContext = True. If you do set useParentContext=True,
will write args into the parent context.
- out. optional. str or dict or list. If you set args or
useParentContext=False, the values in out will be saved from
child pipeline's fresh context into the parent content upon
completion of the child pipeline. Pass a string for a single
key to grab from child context, a list of string for a list
of keys to grab from child context, or a dict where you map
'parent-key-name': 'child-key-name'.
- pipeArg. string. optional. String to pass to the
context_parser - the equivalent to context arg on the
pypyr cli. Only used if skipParse==False.
- raiseError. bool. optional. Defaults to True. If False, log,
but swallow any errors that happen during the invoked
pipeline execution. Swallowing means that the current/parent
pipeline will carry on with the next step even if an error
occurs in the invoked pipeline.
- skipParse. bool. optional. Defaults to True. skip the
context_parser on the invoked pipeline.
- useParentContext. optional. bool. Defaults to True. Pass the
current (i.e parent) pipeline context to the invoked (child)
pipeline.
- loader: str. optional. Absolute name of pipeline loader
module. If not specified will use
pypyr.pypeloaders.fileloader.
- groups. list of str, or str. optional. Step-Groups to run in
pipeline. If you specify a str, will convert it to a single
entry list for you.
- success. str. optional. Step-Group to run on successful
pipeline completion.
- failure. str. optional. Step-Group to run on pipeline error.
If none of groups, success & failure specified, will run the default pypyr
steps, on_success & on_failure sequence.
If groups specified, will only run groups, without a success or failure
sequence, unless you specifically set these also.
Returns:
None
Raises:
pypyr.errors.KeyNotInContextError: if ['pype'] or ['pype']['name']
is missing.
pypyr.errors.KeyInContextHasNoValueError: ['pype']['name'] exists but
is empty.
"""
logger.debug("started")
(pipeline_name, args, out, use_parent_context, pipe_arg, skip_parse,
raise_error, loader, step_groups, success_group,
failure_group) = get_arguments(context)
try:
if use_parent_context:
logger.info("pyping %s, using parent context.", pipeline_name)
if args:
logger.debug("writing args into parent context...")
context.update(args)
pipelinerunner.load_and_run_pipeline(
pipeline_name=pipeline_name,
pipeline_context_input=pipe_arg,
context=context,
parse_input=not skip_parse,
loader=loader,
groups=step_groups,
success_group=success_group,
failure_group=failure_group)
else:
logger.info("pyping %s, without parent context.", pipeline_name)
if args:
child_context = Context(args)
else:
child_context = Context()
child_context.pipeline_name = pipeline_name
child_context.working_dir = context.working_dir
pipelinerunner.load_and_run_pipeline(
pipeline_name=pipeline_name,
pipeline_context_input=pipe_arg,
context=child_context,
parse_input=not skip_parse,
loader=loader,
groups=step_groups,
success_group=success_group,
failure_group=failure_group)
if out:
write_child_context_to_parent(out=out,
parent_context=context,
child_context=child_context)
logger.info("pyped %s.", pipeline_name)
except (ControlOfFlowInstruction, Stop):
# Control-of-Flow/Stop are instructions to go somewhere
# else, not errors per se.
raise
except Exception as ex_info:
# yes, yes, don't catch Exception. Have to, though, in order to swallow
# errs if !raise_error
logger.error("Something went wrong pyping %s. %s: %s", pipeline_name,
type(ex_info).__name__, ex_info)
if raise_error:
logger.debug("Raising original exception to caller.")
raise
else:
logger.debug("raiseError is False. Swallowing error in %s.",
pipeline_name)
logger.debug("done")
def test_nowutc_with_formatting_interpolation():
"""Now gets timestamp with date formatting and pypyr interpolation."""
context = Context({'f': '%Y %w', 'nowUtcIn': '%A {f}'})
nowutc_step.run_step(context)
assert context['nowUtc'] == frozen_timestamp.strftime('%A %Y %w')
def test_nowutc_default_iso():
"""Now gets date in iso format."""
context = Context()
nowutc_step.run_step(context)
assert context['nowUtc'] == frozen_timestamp.isoformat()
def test_pypyr_version_context_out_same_as_in():
"""Context does not mutate."""
context = Context({'test': 'value1'})
pypyr.steps.pypyrversion.run_step(context)
assert context['test'] == 'value1', "context not returned from step."
def test_get_formatted_iterable_nested_with_formatting():
"""Straight deepish copy with formatting."""
# dict containing dict, list, dict-list-dict, tuple, dict-tuple-list, bytes
input_obj = {
'k1':
'v1',
'k2':
'v2_{ctx1}',
'k3':
bytes('v3{ctx1}', encoding='utf-8'),
'k4': [
1, 2, '3_{ctx4}here', {
'key4.1': 'value4.1',
'{ctx2}_key4.2': 'value_{ctx3}_4.2',
'key4.3': {
'4.3.1': '4.3.1value',
'4.3.2': '4.3.2_{ctx1}_value'
}
}
],
'k5': {
'key5.1': 'value5.1',
'key5.2': 'value5.2'
},
'k6': ('six6.1', False, [0, 1, 2], 77, 'six_{ctx1}_end'),
'k7':
'simple string to close 7'
}
context = Context({
'ctx1': 'ctxvalue1',
'ctx2': 'ctxvalue2',
'ctx3': 'ctxvalue3',
'ctx4': 'ctxvalue4',
'contextSetf': {
'output': input_obj,
'{ctx1}': 'substituted key'
}
})
pypyr.steps.contextsetf.run_step(context)
output = context['output']
assert output != input_obj
# verify formatted strings
assert input_obj['k2'] == 'v2_{ctx1}'
assert output['k2'] == 'v2_ctxvalue1'
assert input_obj['k3'] == b'v3{ctx1}'
assert output['k3'] == b'v3{ctx1}'
assert input_obj['k4'][2] == '3_{ctx4}here'
assert output['k4'][2] == '3_ctxvalue4here'
assert input_obj['k4'][3]['{ctx2}_key4.2'] == 'value_{ctx3}_4.2'
assert output['k4'][3]['ctxvalue2_key4.2'] == 'value_ctxvalue3_4.2'
assert input_obj['k4'][3]['key4.3']['4.3.2'] == '4.3.2_{ctx1}_value'
assert output['k4'][3]['key4.3']['4.3.2'] == '4.3.2_ctxvalue1_value'
assert input_obj['k6'][4] == 'six_{ctx1}_end'
assert output['k6'][4] == 'six_ctxvalue1_end'
# verify this was a deep copy - obj refs has to be different for nested
assert id(output['k4']) != id(input_obj['k4'])
assert id(output['k4'][3]['key4.3']) != id(input_obj['k4'][3]['key4.3'])
assert id(output['k5']) != id(input_obj['k5'])
assert id(output['k6']) != id(input_obj['k6'])
assert id(output['k6'][2]) != id(input_obj['k6'][2])
# strings are interned in python, so id is the same
assert id(output['k7']) == id(input_obj['k7'])
output['k7'] = 'mutate 7 on new'
assert input_obj['k7'] == 'simple string to close 7'
assert output['k7'] == 'mutate 7 on new'
assert context['ctxvalue1'] == 'substituted key'
def test_waitfor_substitute_all_args(mock_sleep, mock_service):
"""Successful substitution run with client and method args"""
mock_service.side_effect = [{
'rk1': 'rv1',
'rk2': True
}, {
'rk1': 'rv1',
'rk2': False
}]
context = Context({
'k1': 'v1',
'k2': 'v2',
'k3': 'v3',
'k4': 'v4',
'k5': 'v5',
'k6': 'v6',
'k7': 'v7',
'k8': False,
'k9': 99,
'k10': 77,
'k11': False,
'awsWaitFor': {
'awsClientIn': {
'serviceName': 'service name {k1}',
'methodName': 'method_name {k2}',
'arbKey': 'arb_value {k3}',
'clientArgs': {
'ck1{k4}': 'cv1{k5}',
'ck2': 'cv2'
},
'methodArgs': {
'mk1': 'mv1',
'mk2{k6}': 'mv2{k7}'
}
},
'waitForField': '{rk2}',
'toBe': '{k8}',
'pollInterval': '{k9}',
'maxAttempts': '{k10}',
'errorOnWaitTimeout': '{k11}'
}
})
waitfor_step.run_step(context)
assert len(context) == 13
assert context['k1'] == 'v1'
assert context['awsWaitFor']['awsClientIn'] == {
'serviceName': 'service name {k1}',
'methodName': 'method_name {k2}',
'arbKey': 'arb_value {k3}',
'clientArgs': {
'ck1{k4}': 'cv1{k5}',
'ck2': 'cv2'
},
'methodArgs': {
'mk1': 'mv1',
'mk2{k6}': 'mv2{k7}'
}
}
assert not context['awsWaitForTimedOut']
mock_sleep.assert_called_once_with(99)
mock_service.assert_called_with(
service_name='service name v1',
method_name='method_name v2',
client_args={
'ck1v4': 'cv1v5',
'ck2': 'cv2'
},
operation_args={
'mk1': 'mv1',
'mk2v6': 'mv2v7'
},
)
def test_contextdefault_throws_on_empty_context():
"""Input context must exist."""
with pytest.raises(KeyNotInContextError):
pypyr.steps.default.run_step(Context())
def test_assert_passes_on_assertthis_true():
"""Assert this boolean True passes."""
context = Context({'assert': {'this': True}})
assert_step.run_step(context)
def test_contextsetf_throws_on_empty_context():
"""Context must exist."""
with pytest.raises(KeyNotInContextError):
pypyr.steps.contextsetf.run_step(Context())
def test_assert_passes_on_assertthis_int():
"""Assert this int 1 is True."""
context = Context({'assert': {'this': 1}})
assert_step.run_step(context)
def test_cof_context_key_exists_but_none():
"""A CoF requires context_key in context."""
with pytest.raises(KeyInContextHasNoValueError) as err:
cof_func('blah', None, Context({'key': {'groups': None}}), 'key')
assert str(err.value) == ("key.groups must have a value for step blah")
def test_assert_passes_on_assertthis_arb_negative_int():
"""Assert this non-0 int is True."""
context = Context({'assert': {'this': -55}})
assert_step.run_step(context)
def test_nowutc_with_formatting_date_part():
"""Now gets timestamp with formatting."""
context = Context({'nowUtcIn': '%Y%m%d'})
nowutc_step.run_step(context)
assert context['nowUtc'] == frozen_timestamp.strftime('%Y%m%d')
def test_assert_passes_on_assertthis_float():
"""Assert this non 0 float is True."""
context = Context({'assert': {'this': 3.5}})
assert_step.run_step(context)
def test_step_stop_context_same():
"""Context endures on Stop."""
context = Context({'test': 'value1'})
with pytest.raises(Stop):
pypyr.steps.stop.run_step(context)
assert context['test'] == 'value1', "context not returned from step."
def test_filewritejson_pass_with_payload(mock_makedirs):
"""Success case writes only specific context payload."""
context = Context({
'k1': 'v1',
'fileWriteJson': {
'path': '/arb/blah',
'payload': [
'first',
'second',
{'a': 'b', 'c': 123.45, 'd': [0, 1, 2]},
12,
True
]
}})
with io.StringIO() as out_text:
with patch('pypyr.steps.filewritejson.open',
mock_open()) as mock_output:
mock_output.return_value.write.side_effect = out_text.write
filewrite.run_step(context)
assert context, "context shouldn't be None"
assert len(context) == 2, "context should have 2 items"
assert context['k1'] == 'v1'
assert context['fileWriteJson']['payload'] == [
'first',
'second',
{'a': 'b', 'c': 123.45,
'd': [0, 1, 2]},
12,
True
]
assert context['fileWriteJson'] == {'path': '/arb/blah',
'payload': [
'first',
'second',
{'a': 'b',
'c': 123.45,
'd': [0, 1, 2]},
12,
True
]}
mock_makedirs.assert_called_once_with('/arb', exist_ok=True)
mock_output.assert_called_once_with('/arb/blah', 'w')
# json well formed & new lines + indents are where they should be
assert out_text.getvalue() == ('[\n'
' "first",\n'
' "second",\n'
' {\n'
' "a": "b",\n'
' "c": 123.45,\n'
' "d": [\n'
' 0,\n'
' 1,\n'
' 2\n'
' ]\n'
' },\n'
' 12,\n'
' true\n'
']')
