def test_fetchyaml_pass(): """Relative path to yaml should succeed. Strictly speaking not a unit test. """ context = Context({ 'ok1': 'ov1', 'fetchYaml': { 'path': './tests/testfiles/dict.yaml' } }) filefetcher.run_step(context) assert context, "context shouldn't be None" assert len(context) == 7, "context should have 7 items" assert context['ok1'] == 'ov1' assert context['fetchYaml']['path'] == './tests/testfiles/dict.yaml' assert context['key2'] == 'value2', "key2 should be value2" assert len(context['key4']['k42']) == 3, "3 items in k42" assert 'k42list2' in context['key4']['k42'], "k42 containts k42list2" assert context['key4']['k43'], "k43 is True" assert context['key4']['k44'] == 77, "k44 is 77" assert len(context['key5']) == 2, "2 items in key5"
def test_pycode_replace_objects(): """Rebind mutable and immutable objects in eval scope.""" pycode = """\ context['alist'] = [0, 1, 2] context['adict'] = {'a': 'b', 'c': 'd'} context['anint'] = 456 context['mutate_me'].append(12) context['astring'] = 'updated' """ context = Context({'alist': [0, 1], 'adict': {'a': 'b'}, 'anint': 123, 'mutate_me': [10, 11], 'astring': 'a string', 'pycode': pycode}) pypyr.steps.py.run_step(context) assert context == {'alist': [0, 1, 2], 'adict': {'a': 'b', 'c': 'd'}, 'anint': 456, 'mutate_me': [10, 11, 12], 'astring': 'updated', 'pycode': pycode}
def test_pype_get_arguments_group_str(): """Parse group as str input from context.""" context = Context({'pype': { 'name': 'pipe name', 'groups': 'gr', }}) (pipeline_name, args, out, use_parent_context, pipe_arg, skip_parse, raise_error, loader, groups, success_group, failure_group) = pype.get_arguments(context) assert pipeline_name == 'pipe name' assert not args assert not out assert use_parent_context assert isinstance(use_parent_context, bool) assert skip_parse assert isinstance(skip_parse, bool) assert raise_error assert isinstance(raise_error, bool) assert loader is None assert groups == ['gr'] assert success_group is None assert failure_group is None
def test_envget_pass_with_substitutions(): """Env get success case with string substitutions.""" os.environ['ARB_DELETE_ME1'] = 'arb value from $ENV ARB_DELETE_ME1' context = Context({ 'key1': 'value1', 'key2': 'value2', 'env_val1': 'ARB_DELETE_ME1', 'env_val2': 'ARB_DELETE_ME2', 'default_val': 'blah', 'key_val': 'key3', 'envGet': [{ 'env': '{env_val1}', 'key': '{key_val}', 'default': 'blah' }, { 'env': '{env_val2}', 'key': 'key4', 'default': '{default_val}' }] }) pypyr.steps.envget.run_step(context) del os.environ['ARB_DELETE_ME1'] assert context['key1'] == 'value1' assert context['key2'] == 'value2' assert context['key3'] == 'arb value from $ENV ARB_DELETE_ME1' assert context['key4'] == 'blah'
def test_aws_wait_pass_no_args(mock_waiter): """Successful run with no client args""" context = Context({ 'k1': 'v1', 'awsWaitIn': { 'serviceName': 'service name', 'waiterName': 'waiter_name', 'arbKey': 'arb_value' }}) wait.run_step(context) assert len(context) == 2 assert context['k1'] == 'v1' assert context['awsWaitIn'] == { 'serviceName': 'service name', 'waiterName': 'waiter_name', 'arbKey': 'arb_value' } mock_waiter.assert_called_once_with(service_name='service name', waiter_name='waiter_name', waiter_args=None, wait_args=None, )
def test_envget_pass(): """envget success case""" os.environ['ARB_DELETE_ME1'] = 'arb value from $ENV ARB_DELETE_ME1' os.environ['ARB_DELETE_ME2'] = 'arb value from $ENV ARB_DELETE_ME2' context = Context({ 'key1': 'value1', 'key2': 'value2', 'key3': 'value3', 'envGet': { 'key2': 'ARB_DELETE_ME1', 'key4': 'ARB_DELETE_ME2' } }) pypyr.steps.env.env_get(context) assert context['key1'] == 'value1' assert context['key2'] == 'arb value from $ENV ARB_DELETE_ME1' assert context['key3'] == 'value3' assert context['key4'] == 'arb value from $ENV ARB_DELETE_ME2' del os.environ['ARB_DELETE_ME1'] del os.environ['ARB_DELETE_ME2']
def test_pyimport(): """Import namespace saved to context.""" source = """\ import math import tests.arbpack.arbmod import tests.arbpack.arbmod2 from tests.arbpack.arbmultiattr import arb_attr, arb_func as y """ context = Context({'pyImport': source}) pyimport.run_step(context) ns = context._pystring_globals len(ns) == 4 assert ns['math'].sqrt(4) == 2 # no return value but shouldn't raise not found. ns['tests'].arbpack.arbmod.arbmod_attribute() assert ns['arb_attr'] == 123.456 assert ns['y']('ab3') == 'ab3' # parent did not import anything NOT specified. # tests.arbpack.arbstep exists but wasn't specified for import. assert not hasattr(ns['tests'].arbpack, 'arbstep')
def test_pype_get_args_no_parent_context(): """If args set use_parent_context should default False.""" context = Context({'pype': { 'name': 'pipe name', 'args': { 'a': 'b' }, }}) (pipeline_name, args, out, use_parent_context, pipe_arg, skip_parse, raise_error, loader, groups, success_group, failure_group) = pype.get_arguments(context) assert pipeline_name == 'pipe name' assert args == {'a': 'b'} assert out is None assert not use_parent_context assert pipe_arg is None assert skip_parse assert raise_error assert not loader assert not groups assert not success_group assert not failure_group
def test_assert_raises_on_assertthis_not_equals_dict_to_dict_substitutions(): """Assert this string does not equal assertEquals dict.""" context = Context({ 'k1': 'v1', 'k2': 'v2', 'assert': { 'this': { 'k1': 1, 'k2': [2, '{k1}'], 'k3': False }, 'equals': { 'k1': 1, 'k2': [2, '{k2}'], 'k3': False } } }) with pytest.raises(AssertionError) as err_info: assert_step.run_step(context) assert str( err_info.value) == ("assert assert['this'] is of type dict and does " "not equal assert['equals'] of type dict.")
def test_pathcheck_single(mock_glob): """Single path ok.""" context = Context({'ok1': 'ov1', 'pathCheck': './arb/x'}) mock_glob.return_value = ['./foundfile'] with patch_logger('pypyr.steps.pathcheck', logging.INFO) as mock_logger_info: pathchecker.run_step(context) mock_logger_info.assert_called_once_with('checked 1 path(s) and found 1') assert context, "context shouldn't be None" assert len(context) == 3, "context should have 3 items" assert context['ok1'] == 'ov1' assert context['pathCheck'] == './arb/x' assert context["pathCheckOut"] == { './arb/x': { 'exists': True, 'count': 1, 'found': ['./foundfile'] } } mock_glob.assert_called_once_with('./arb/x')
def test_pype_use_parent_context_with_swallow(mock_run_pipeline): """pype swallowing error in child pipeline.""" context = Context({ 'pype': { 'name': 'pipe name', 'pipeArg': 'argument here', 'useParentContext': True, 'skipParse': True, 'raiseError': False } }) logger = logging.getLogger('pypyr.steps.pype') with patch.object(logger, 'error') as mock_logger_error: pype.run_step(context) mock_run_pipeline.assert_called_once_with( pipeline_name='pipe name', pipeline_context_input='argument here', context=context, parse_input=False) mock_logger_error.assert_called_once_with( 'Something went wrong pyping pipe name. RuntimeError: whoops')
def test_tar_only_calls_archive(): """Only calls archive if only archive specified.""" context = Context({ 'key1': 'value1', 'key2': 'value2', 'key3': 'value3', 'tar': { 'archive': [{ 'in': 'key2', 'out': 'ARB_GET_ME1' }, { 'in': 'key4', 'out': 'ARB_GET_ME2' }] } }) with patch.multiple('pypyr.steps.tar', tar_archive=DEFAULT, tar_extract=DEFAULT) as mock_tar: pypyr.steps.tar.run_step(context) mock_tar['tar_extract'].assert_not_called() mock_tar['tar_archive'].assert_called_once()
def test_waitfor_pass_all_args(mock_sleep, mock_service): """Successful run with client and method args pass on #2.""" mock_service.side_effect = [ { 'rk1': 'rv1', 'rk2': 'rv2' }, # 1 { 'rk1': 'rv1', 'rk2': False }, # 2 { 'rk1': 'rv1', 'rk2': 'rv2' }, # 3 ] context = Context({ 'k1': 'v1', 'awsWaitFor': { 'awsClientIn': { 'serviceName': 'service name', 'methodName': 'method_name', 'arbKey': 'arb_value', 'clientArgs': { 'ck1': 'cv1', 'ck2': 'cv2' }, 'methodArgs': { 'mk1': 'mv1', 'mk2': 'mv2' } }, 'waitForField': '{rk2}', 'toBe': False, 'pollInterval': 99 } }) waitfor_step.run_step(context) assert len(context) == 3 assert context['k1'] == 'v1' assert context['awsWaitFor']['awsClientIn'] == { 'serviceName': 'service name', 'methodName': 'method_name', 'arbKey': 'arb_value', 'clientArgs': { 'ck1': 'cv1', 'ck2': 'cv2' }, 'methodArgs': { 'mk1': 'mv1', 'mk2': 'mv2' } } assert not context['awsWaitForTimedOut'] mock_sleep.assert_called_once_with(99) mock_service.assert_called_with( service_name='service name', method_name='method_name', client_args={ 'ck1': 'cv1', 'ck2': 'cv2' }, operation_args={ 'mk1': 'mv1', 'mk2': 'mv2' }, )
def test_waitfor_pass_client_args(mock_sleep, mock_service): """Successful run with client args pass on 3.""" mock_service.side_effect = [ { 'rk1': 'rv1', 'rk2': 'rv2' }, # 1 { 'rk1': 'rv1', 'rk2': 'rv2' }, # 2 { 'rk1': 'rv1', 'rk2': 'xxx' }, # 3 { 'rk1': 'rv1', 'rk2': 'rv2' } # 4 ] context = Context({ 'k1': 'v1', 'awsWaitFor': { 'awsClientIn': { 'serviceName': 'service name', 'methodName': 'method_name', 'arbKey': 'arb_value', 'clientArgs': { 'ck1': 'cv1', 'ck2': 'cv2' } }, 'waitForField': '{rk2}', 'toBe': 'xxx' } }) waitfor_step.run_step(context) assert len(context) == 3 assert context['k1'] == 'v1' assert context['awsWaitFor']['awsClientIn'] == { 'serviceName': 'service name', 'methodName': 'method_name', 'arbKey': 'arb_value', 'clientArgs': { 'ck1': 'cv1', 'ck2': 'cv2' } } assert not context['awsWaitForTimedOut'] mock_service.assert_called_with( service_name='service name', method_name='method_name', client_args={ 'ck1': 'cv1', 'ck2': 'cv2' }, operation_args=None, ) mock_sleep.call_count == 3 mock_sleep.assert_called_with(30)
def run_step(context): """Run another pipeline from this step. The parent pipeline is the current, executing pipeline. The invoked, or child pipeline is the pipeline you are calling from this step. Args: context: dictionary-like pypyr.context.Context. context is mandatory. Uses the following context keys in context: - pype - name. mandatory. str. Name of pipeline to execute. This {name}.yaml must exist in the working directory/pipelines dir. - args. optional. dict. Create the context of the called pipeline with these keys & values. If args specified, will not pass the parent context unless you explicitly set useParentContext = True. If you do set useParentContext=True, will write args into the parent context. - out. optional. str or dict or list. If you set args or useParentContext=False, the values in out will be saved from child pipeline's fresh context into the parent content upon completion of the child pipeline. Pass a string for a single key to grab from child context, a list of string for a list of keys to grab from child context, or a dict where you map 'parent-key-name': 'child-key-name'. - pipeArg. string. optional. String to pass to the context_parser - the equivalent to context arg on the pypyr cli. Only used if skipParse==False. - raiseError. bool. optional. Defaults to True. If False, log, but swallow any errors that happen during the invoked pipeline execution. Swallowing means that the current/parent pipeline will carry on with the next step even if an error occurs in the invoked pipeline. - skipParse. bool. optional. Defaults to True. skip the context_parser on the invoked pipeline. - useParentContext. optional. bool. Defaults to True. Pass the current (i.e parent) pipeline context to the invoked (child) pipeline. - loader: str. optional. Absolute name of pipeline loader module. If not specified will use pypyr.pypeloaders.fileloader. - groups. list of str, or str. optional. Step-Groups to run in pipeline. If you specify a str, will convert it to a single entry list for you. - success. str. optional. Step-Group to run on successful pipeline completion. - failure. str. optional. Step-Group to run on pipeline error. If none of groups, success & failure specified, will run the default pypyr steps, on_success & on_failure sequence. If groups specified, will only run groups, without a success or failure sequence, unless you specifically set these also. Returns: None Raises: pypyr.errors.KeyNotInContextError: if ['pype'] or ['pype']['name'] is missing. pypyr.errors.KeyInContextHasNoValueError: ['pype']['name'] exists but is empty. """ logger.debug("started") (pipeline_name, args, out, use_parent_context, pipe_arg, skip_parse, raise_error, loader, step_groups, success_group, failure_group) = get_arguments(context) try: if use_parent_context: logger.info("pyping %s, using parent context.", pipeline_name) if args: logger.debug("writing args into parent context...") context.update(args) pipelinerunner.load_and_run_pipeline( pipeline_name=pipeline_name, pipeline_context_input=pipe_arg, context=context, parse_input=not skip_parse, loader=loader, groups=step_groups, success_group=success_group, failure_group=failure_group) else: logger.info("pyping %s, without parent context.", pipeline_name) if args: child_context = Context(args) else: child_context = Context() child_context.pipeline_name = pipeline_name child_context.working_dir = context.working_dir pipelinerunner.load_and_run_pipeline( pipeline_name=pipeline_name, pipeline_context_input=pipe_arg, context=child_context, parse_input=not skip_parse, loader=loader, groups=step_groups, success_group=success_group, failure_group=failure_group) if out: write_child_context_to_parent(out=out, parent_context=context, child_context=child_context) logger.info("pyped %s.", pipeline_name) except (ControlOfFlowInstruction, Stop): # Control-of-Flow/Stop are instructions to go somewhere # else, not errors per se. raise except Exception as ex_info: # yes, yes, don't catch Exception. Have to, though, in order to swallow # errs if !raise_error logger.error("Something went wrong pyping %s. %s: %s", pipeline_name, type(ex_info).__name__, ex_info) if raise_error: logger.debug("Raising original exception to caller.") raise else: logger.debug("raiseError is False. Swallowing error in %s.", pipeline_name) logger.debug("done")
def test_nowutc_with_formatting_interpolation(): """Now gets timestamp with date formatting and pypyr interpolation.""" context = Context({'f': '%Y %w', 'nowUtcIn': '%A {f}'}) nowutc_step.run_step(context) assert context['nowUtc'] == frozen_timestamp.strftime('%A %Y %w')
def test_nowutc_default_iso(): """Now gets date in iso format.""" context = Context() nowutc_step.run_step(context) assert context['nowUtc'] == frozen_timestamp.isoformat()
def test_pypyr_version_context_out_same_as_in(): """Context does not mutate.""" context = Context({'test': 'value1'}) pypyr.steps.pypyrversion.run_step(context) assert context['test'] == 'value1', "context not returned from step."
def test_get_formatted_iterable_nested_with_formatting(): """Straight deepish copy with formatting.""" # dict containing dict, list, dict-list-dict, tuple, dict-tuple-list, bytes input_obj = { 'k1': 'v1', 'k2': 'v2_{ctx1}', 'k3': bytes('v3{ctx1}', encoding='utf-8'), 'k4': [ 1, 2, '3_{ctx4}here', { 'key4.1': 'value4.1', '{ctx2}_key4.2': 'value_{ctx3}_4.2', 'key4.3': { '4.3.1': '4.3.1value', '4.3.2': '4.3.2_{ctx1}_value' } } ], 'k5': { 'key5.1': 'value5.1', 'key5.2': 'value5.2' }, 'k6': ('six6.1', False, [0, 1, 2], 77, 'six_{ctx1}_end'), 'k7': 'simple string to close 7' } context = Context({ 'ctx1': 'ctxvalue1', 'ctx2': 'ctxvalue2', 'ctx3': 'ctxvalue3', 'ctx4': 'ctxvalue4', 'contextSetf': { 'output': input_obj, '{ctx1}': 'substituted key' } }) pypyr.steps.contextsetf.run_step(context) output = context['output'] assert output != input_obj # verify formatted strings assert input_obj['k2'] == 'v2_{ctx1}' assert output['k2'] == 'v2_ctxvalue1' assert input_obj['k3'] == b'v3{ctx1}' assert output['k3'] == b'v3{ctx1}' assert input_obj['k4'][2] == '3_{ctx4}here' assert output['k4'][2] == '3_ctxvalue4here' assert input_obj['k4'][3]['{ctx2}_key4.2'] == 'value_{ctx3}_4.2' assert output['k4'][3]['ctxvalue2_key4.2'] == 'value_ctxvalue3_4.2' assert input_obj['k4'][3]['key4.3']['4.3.2'] == '4.3.2_{ctx1}_value' assert output['k4'][3]['key4.3']['4.3.2'] == '4.3.2_ctxvalue1_value' assert input_obj['k6'][4] == 'six_{ctx1}_end' assert output['k6'][4] == 'six_ctxvalue1_end' # verify this was a deep copy - obj refs has to be different for nested assert id(output['k4']) != id(input_obj['k4']) assert id(output['k4'][3]['key4.3']) != id(input_obj['k4'][3]['key4.3']) assert id(output['k5']) != id(input_obj['k5']) assert id(output['k6']) != id(input_obj['k6']) assert id(output['k6'][2]) != id(input_obj['k6'][2]) # strings are interned in python, so id is the same assert id(output['k7']) == id(input_obj['k7']) output['k7'] = 'mutate 7 on new' assert input_obj['k7'] == 'simple string to close 7' assert output['k7'] == 'mutate 7 on new' assert context['ctxvalue1'] == 'substituted key'
def test_waitfor_substitute_all_args(mock_sleep, mock_service): """Successful substitution run with client and method args""" mock_service.side_effect = [{ 'rk1': 'rv1', 'rk2': True }, { 'rk1': 'rv1', 'rk2': False }] context = Context({ 'k1': 'v1', 'k2': 'v2', 'k3': 'v3', 'k4': 'v4', 'k5': 'v5', 'k6': 'v6', 'k7': 'v7', 'k8': False, 'k9': 99, 'k10': 77, 'k11': False, 'awsWaitFor': { 'awsClientIn': { 'serviceName': 'service name {k1}', 'methodName': 'method_name {k2}', 'arbKey': 'arb_value {k3}', 'clientArgs': { 'ck1{k4}': 'cv1{k5}', 'ck2': 'cv2' }, 'methodArgs': { 'mk1': 'mv1', 'mk2{k6}': 'mv2{k7}' } }, 'waitForField': '{rk2}', 'toBe': '{k8}', 'pollInterval': '{k9}', 'maxAttempts': '{k10}', 'errorOnWaitTimeout': '{k11}' } }) waitfor_step.run_step(context) assert len(context) == 13 assert context['k1'] == 'v1' assert context['awsWaitFor']['awsClientIn'] == { 'serviceName': 'service name {k1}', 'methodName': 'method_name {k2}', 'arbKey': 'arb_value {k3}', 'clientArgs': { 'ck1{k4}': 'cv1{k5}', 'ck2': 'cv2' }, 'methodArgs': { 'mk1': 'mv1', 'mk2{k6}': 'mv2{k7}' } } assert not context['awsWaitForTimedOut'] mock_sleep.assert_called_once_with(99) mock_service.assert_called_with( service_name='service name v1', method_name='method_name v2', client_args={ 'ck1v4': 'cv1v5', 'ck2': 'cv2' }, operation_args={ 'mk1': 'mv1', 'mk2v6': 'mv2v7' }, )
def test_contextdefault_throws_on_empty_context(): """Input context must exist.""" with pytest.raises(KeyNotInContextError): pypyr.steps.default.run_step(Context())
def test_assert_passes_on_assertthis_true(): """Assert this boolean True passes.""" context = Context({'assert': {'this': True}}) assert_step.run_step(context)
def test_contextsetf_throws_on_empty_context(): """Context must exist.""" with pytest.raises(KeyNotInContextError): pypyr.steps.contextsetf.run_step(Context())
def test_assert_passes_on_assertthis_int(): """Assert this int 1 is True.""" context = Context({'assert': {'this': 1}}) assert_step.run_step(context)
def test_cof_context_key_exists_but_none(): """A CoF requires context_key in context.""" with pytest.raises(KeyInContextHasNoValueError) as err: cof_func('blah', None, Context({'key': {'groups': None}}), 'key') assert str(err.value) == ("key.groups must have a value for step blah")
def test_assert_passes_on_assertthis_arb_negative_int(): """Assert this non-0 int is True.""" context = Context({'assert': {'this': -55}}) assert_step.run_step(context)
def test_nowutc_with_formatting_date_part(): """Now gets timestamp with formatting.""" context = Context({'nowUtcIn': '%Y%m%d'}) nowutc_step.run_step(context) assert context['nowUtc'] == frozen_timestamp.strftime('%Y%m%d')
def test_assert_passes_on_assertthis_float(): """Assert this non 0 float is True.""" context = Context({'assert': {'this': 3.5}}) assert_step.run_step(context)
def test_step_stop_context_same(): """Context endures on Stop.""" context = Context({'test': 'value1'}) with pytest.raises(Stop): pypyr.steps.stop.run_step(context) assert context['test'] == 'value1', "context not returned from step."
def test_filewritejson_pass_with_payload(mock_makedirs): """Success case writes only specific context payload.""" context = Context({ 'k1': 'v1', 'fileWriteJson': { 'path': '/arb/blah', 'payload': [ 'first', 'second', {'a': 'b', 'c': 123.45, 'd': [0, 1, 2]}, 12, True ] }}) with io.StringIO() as out_text: with patch('pypyr.steps.filewritejson.open', mock_open()) as mock_output: mock_output.return_value.write.side_effect = out_text.write filewrite.run_step(context) assert context, "context shouldn't be None" assert len(context) == 2, "context should have 2 items" assert context['k1'] == 'v1' assert context['fileWriteJson']['payload'] == [ 'first', 'second', {'a': 'b', 'c': 123.45, 'd': [0, 1, 2]}, 12, True ] assert context['fileWriteJson'] == {'path': '/arb/blah', 'payload': [ 'first', 'second', {'a': 'b', 'c': 123.45, 'd': [0, 1, 2]}, 12, True ]} mock_makedirs.assert_called_once_with('/arb', exist_ok=True) mock_output.assert_called_once_with('/arb/blah', 'w') # json well formed & new lines + indents are where they should be assert out_text.getvalue() == ('[\n' ' "first",\n' ' "second",\n' ' {\n' ' "a": "b",\n' ' "c": 123.45,\n' ' "d": [\n' ' 0,\n' ' 1,\n' ' 2\n' ' ]\n' ' },\n' ' 12,\n' ' true\n' ']')