def get_waiter_args(context):
"""Gets required args from context for this step.
Args:
context - dict. context.
Returns:
tuple(client_in, service_name, waiter_name)
Raises:
pypyr.errors.KeyNotInContextError: Required key missing in context.
pypyr.errors.KeyInContextHasNoValueError: Required key exists but is
empty or None.
"""
try:
client_in = context['awsWaitIn']
service_name = client_in['serviceName']
waiter_name = client_in['waiterName']
except KeyError as err:
raise KeyNotInContextError(
"awsWaitIn missing required key for pypyraws.steps.wait: "
f"{err}"
) from err
# of course, if They went and made it a bool and True this will pass.
if not (service_name and service_name.strip()):
raise KeyInContextHasNoValueError(
'serviceName required in awsWaitIn for pypyraws.steps.wait')
if not (waiter_name and waiter_name.strip()):
raise KeyInContextHasNoValueError(
'waiterName required in awsWaitIn for pypyraws.steps.wait')
return client_in, service_name, waiter_name
def get_awsclient_args(context, calling_module_name):
"""Get required args from context for awsClientIn type steps.
Args:
context: pypyr.context.Context.
calling_module_name: string. This is just to make a friendly error msg
should something go wrong.
Returns:
tuple(client_in, service_name, method_name)
Raises:
pypyr.errors.KeyNotInContextError: Required key missing in context.
pypyr.errors.KeyInContextHasNoValueError: Required key exists but is
empty or None.
"""
try:
client_in = context['awsClientIn']
service_name = client_in['serviceName']
method_name = client_in['methodName']
except KeyError as err:
raise KeyNotInContextError(
f"awsClientIn missing required key for {calling_module_name}: "
f"{err}") from err
if not (service_name and service_name.strip()):
raise KeyInContextHasNoValueError(
f'serviceName required in awsClientIn for {calling_module_name}')
if not (method_name and method_name.strip()):
raise KeyInContextHasNoValueError(
f'methodName required in awsClientIn for {calling_module_name}')
return client_in, service_name, method_name
def test_key_in_context_has_no_value_error_raises():
"""Key not in context value error raises with correct message."""
# confirm subclassed from pypyr root error
assert isinstance(KeyInContextHasNoValueError(), PypyrError)
assert isinstance(KeyNotInContextError(), ContextError)
with pytest.raises(KeyInContextHasNoValueError) as err_info:
raise KeyInContextHasNoValueError("this is error text right here")
assert str(err_info.value) == "this is error text right here"
def run_step(context):
"""Get paths from a glob input.
Do note that this will returns files AND directories that match the glob.
No tilde expansion is done, but *, ?, and character ranges expressed with
[] will be correctly matched.
Escape all special characters ('?', '*' and '['). For a literal match, wrap
the meta-characters in brackets. For example, '[?]' matches the character
'?'.
If passing in an iterable of paths, will expand matches for each path in
the iterable. The function will return all the matches for each path
glob expression combined into a single list.
If no matches found, writes empty list [] to globOut.
Args:
context: pypyr.context.Context. Mandatory.
The following context key is mandatory:
- glob. str or list. Single path, or list of paths.
All inputs support pypyr formatting expressions.
Returns:
None. updates context ['globOut'] with list of resolved paths as
strings.
"""
logger.debug("started")
context.assert_key_has_value(key='glob', caller=__name__)
paths = context.get_formatted('glob')
if isinstance(paths, list):
if not paths or any(not p for p in paths):
raise KeyInContextHasNoValueError("The glob list has an empty str")
in_count = len(paths)
else:
if not paths:
raise KeyInContextHasNoValueError("The glob path is an empty str")
in_count = 1
context['globOut'] = pypyr.utils.filesystem.get_glob(paths)
logger.info("glob checked %s globs and saved "
"%s paths to globOut", in_count, len(context['globOut']))
logger.debug("done")
def assert_key_has_value(obj, key, caller, parent=None):
"""Assert key exists in obj and has value.
Does not eval on truthy. Empty string or 0 counts as a value.
Error messages are structured as if obj is a pypyr Context.
Args:
obj (mapping): object to check for key.
key (any valid key type): key to check in obj
caller (string): calling function name - this used to construct
error messages. Tip: use .__name__
parent (string): parent key name. Used to construct error messages to
indicate the name of obj in context.
Raises:
ContextError: if obj is None or not iterable.
KeyInContextHasNoValueError: context[key] is None
KeyNotInContextError: Key doesn't exist
"""
assert_key_exists(obj, key, caller, parent)
if obj[key] is None:
if parent:
msg = (f"context[{parent!r}][{key!r}] must have a value for "
f"{caller}.")
else:
msg = f"context[{key!r}] must have a value for {caller}."
raise KeyInContextHasNoValueError(msg)
def assert_key_type_value(self,
context_item,
caller,
extra_error_text=''):
"""Assert that keys exist of right type and has a value.
Args:
context_item: ContextItemInfo tuple
caller: string. calling function name - this used to construct
error messages
extra_error_text: append to end of error message.
Raises:
AssertionError: if context_item None.
KeyNotInContextError: Key doesn't exist
KeyInContextHasNoValueError: context[key] is None or the wrong
type.
"""
assert context_item, ("context_item parameter must be specified.")
if extra_error_text is None or extra_error_text == '':
append_error_text = ''
else:
append_error_text = f' {extra_error_text}'
if not context_item.key_in_context:
raise KeyNotInContextError(f'{caller} couldn\'t find '
f'{context_item.key} in context.'
f'{append_error_text}')
if not context_item.has_value:
raise KeyInContextHasNoValueError(
f'{caller} found {context_item.key} in '
f'context but it doesn\'t have a value.'
f'{append_error_text}')
if not context_item.is_expected_type:
raise KeyInContextHasNoValueError(
f'{caller} found {context_item.key} in context, but it\'s '
f'not a {context_item.expected_type}.'
f'{append_error_text}')
def get_awsclient_args(input_dict, calling_module_name):
"""Get required args from context for awsClientIn type steps.
Doesn't do any formatting. You gotta format before you get here.
Args:
input_dict (dict): dict-like structure containing awsClientIn key.
calling_module_name: string. This is just to make a friendly error msg
should something go wrong.
Returns:
tuple(service_name, method_name, client_args, method_args)
Raises:
pypyr.errors.KeyNotInContextError: Required key missing in context.
pypyr.errors.KeyInContextHasNoValueError: Required key exists but is
empty or None.
"""
try:
service_name = input_dict['serviceName']
method_name = input_dict['methodName']
except KeyError as err:
raise KeyNotInContextError(
f"awsClientIn missing required key for {calling_module_name}: "
f"{err}"
) from err
if not (service_name and service_name.strip()):
raise KeyInContextHasNoValueError(
f'serviceName required in awsClientIn for {calling_module_name}')
if not (method_name and method_name.strip()):
raise KeyInContextHasNoValueError(
f'methodName required in awsClientIn for {calling_module_name}')
client_args = input_dict.get('clientArgs', None)
method_args = input_dict.get('methodArgs', None)
return service_name, method_name, client_args, method_args
def get_arguments(context):
"""Parse arguments for pype from context and assign default values.
Args:
context: pypyr.context.Context. context is mandatory.
Returns:
tuple (pipeline_name, #str
use_parent_context, #bool
pipe_arg, #str
skip_parse, #bool
raise_error #bool
)
Raises:
pypyr.errors.KeyNotInContextError: if ['pype']['name'] is missing.
pypyr.errors.KeyInContextHasNoValueError: if ['pype']['name'] exists but
is None.
"""
context.assert_key_has_value(key='pype', caller=__name__)
pype = context['pype']
try:
pipeline_name = pype['name']
if pipeline_name is None:
raise KeyInContextHasNoValueError(
"pypyr.steps.pype ['pype']['name'] exists but is empty.")
except KeyError as err:
raise KeyNotInContextError(
"pypyr.steps.pype missing 'name' in the 'pype' context item. "
"You need to specify the pipeline name to run another "
"pipeline.") from err
use_parent_context = pype.get('useParentContext', True)
pipe_arg = pype.get('pipeArg', None)
skip_parse = pype.get('skipParse', True)
raise_error = pype.get('raiseError', True)
return (pipeline_name, use_parent_context, pipe_arg, skip_parse,
raise_error)
def assert_key_has_value(self, key, caller):
"""Assert that context contains key which also has a value.
Args:
key: validate this key exists in context AND has a value that isn't
None.
caller: string. calling function name - this used to construct
error messages
Raises:
KeyNotInContextError: Key doesn't exist
KeyInContextHasNoValueError: context[key] is None
AssertionError: if key is None
"""
assert key, ("key parameter must be specified.")
self.assert_key_exists(key, caller)
if self[key] is None:
raise KeyInContextHasNoValueError(
f"context['{key}'] must have a value for {caller}.")
def assert_child_key_has_value(self, parent, child, caller):
"""Assert that context contains key that has child which has a value.
Args:
parent: parent key
child: validate this sub-key of parent exists AND isn't None.
caller: string. calling function name - this used to construct
error messages
Raises:
KeyNotInContextError: Key doesn't exist
KeyInContextHasNoValueError: context[key] is None
AssertionError: if key is None
"""
assert parent, ("parent parameter must be specified.")
assert child, ("child parameter must be specified.")
self.assert_key_has_value(parent, caller)
try:
child_exists = child in self[parent]
except TypeError as err:
# This happens if parent isn't iterable
raise ContextError(
f"context['{parent}'] must be iterable and contain '{child}' "
f"for {caller}. {err}") from err
if child_exists:
if self[parent][child] is None:
raise KeyInContextHasNoValueError(
f"context['{parent}']['{child}'] must have a value for "
f"{caller}.")
else:
raise KeyNotInContextError(
f"context['{parent}']['{child}'] doesn't "
f"exist. It must exist for {caller}.")
def run_step(context):
"""Check if a file or directory path exists.
Args:
context: pypyr.context.Context. Mandatory.
The following context key must exist
- pathsToCheck. str/path-like or list of str/paths.
Path to file on disk to check.
All inputs support formatting expressions. Supports globs.
This step creates pathCheckOut in context, containing the results of the
path check operation.
pathCheckOut:
'inpath':
exists: true # bool. True if path exists.
count: 0 # int. Number of files found for in path.
found: ['path1', 'path2'] # list of strings. Paths of files found.
[count] is 0 if no files found. If you specified a single input
path to check and it exists, it's going to be 1. If you specified multiple
in paths or a glob expression that found more than 1 result, well, take a
guess.
[found] is a list of all the paths found for the [inpath]. If you passed
in a glob or globs, will contain the globs found for [inpath].
This means you can do an existence evaluation like this in a formatting
expression: '{pathCheckOut[inpathhere][exists]}'
Returns:
None. updates context arg.
Raises:
pypyr.errors.KeyNotInContextError: pathExists missing in context.
pypyr.errors.KeyInContextHasNoValueError: pathCheck exists but is None.
"""
logger.debug("started")
context.assert_key_has_value(key='pathCheck', caller=__name__)
paths_to_check = context['pathCheck']
if not paths_to_check:
raise KeyInContextHasNoValueError("context['pathCheck'] must have a "
f"value for {__name__}.")
# pathsToCheck can be a string or a list in case there are multiple paths
if isinstance(paths_to_check, list):
check_me = paths_to_check
else:
# assuming it's a str/path at this point
check_me = [paths_to_check]
out = {}
total_found = 0
for path in check_me:
logger.debug("checking path: %s", path)
formatted_path = context.get_formatted_value(path)
found_paths = pypyr.utils.filesystem.get_glob(formatted_path)
no_of_paths = len(found_paths)
out[path] = {
'exists': no_of_paths > 0,
'count': no_of_paths,
'found': found_paths
}
total_found = total_found + no_of_paths
context['pathCheckOut'] = out
logger.info(f'checked {len(out)} path(s) and found {total_found}')
logger.debug("done")
def get_arguments(context):
"""Parse arguments for pype from context and assign default values.
Args:
context: pypyr.context.Context. context is mandatory.
Returns:
tuple (pipeline_name, #str
args, #dict
out, #str or dict or list
use_parent_context, #bool
pipe_arg, #str
skip_parse, #bool
raise_error #bool
groups #list of str
success_group #str
failure_group #str
)
Raises:
pypyr.errors.KeyNotInContextError: if ['pype']['name'] is missing.
pypyr.errors.KeyInContextHasNoValueError: if ['pype']['name'] exists but
is None.
"""
context.assert_key_has_value(key='pype', caller=__name__)
pype = context.get_formatted('pype')
try:
pipeline_name = pype['name']
if pipeline_name is None:
raise KeyInContextHasNoValueError(
"pypyr.steps.pype ['pype']['name'] exists but is empty.")
except KeyError as err:
raise KeyNotInContextError(
"pypyr.steps.pype missing 'name' in the 'pype' context item. "
"You need to specify the pipeline name to run another "
"pipeline.") from err
args = pype.get('args', None)
if args is not None and not isinstance(args, dict):
raise ContextError(
"pypyr.steps.pype 'args' in the 'pype' context item "
"must be a dict.")
if args and 'useParentContext' not in pype:
use_parent_context = False
else:
use_parent_context = pype.get('useParentContext', True)
out = pype.get('out', None)
if out and use_parent_context:
raise ContextError(
"pypyr.steps.pype pype.out is only relevant if useParentContext "
"= False. If you're using the parent context, no need to have out "
"args since their values will already be in context. If you're "
"NOT using parent context and you've specified pype.args, just "
"leave off the useParentContext key and it'll default to False "
"under the hood, or set it to False yourself if you keep it in.")
pipe_arg = pype.get('pipeArg', None)
skip_parse = pype.get('skipParse', True)
raise_error = pype.get('raiseError', True)
loader = pype.get('loader', None)
groups = pype.get('groups', None)
if isinstance(groups, str):
groups = [groups]
success_group = pype.get('success', None)
failure_group = pype.get('failure', None)
return (pipeline_name, args, out, use_parent_context, pipe_arg, skip_parse,
raise_error, loader, groups, success_group, failure_group)
def control_of_flow_instruction(name, instruction_type, context, context_key):
"""Run a control of flow instruction.
The step config in the context dict looks like this:
<<instruction-name>>: <<cmd string>>. Mandatory.
OR, as a dict
<<instruction-name:
groups: <<str>> or <<list of str>> - mandatory.
success: <<str>>
failure: <<str>>
Args:
name: Unique name for step. Likely __name__ of calling step.
instruction_type: Type - must inherit from
pypyr.errors.ControlOfFlowInstruction
context: pypyr.context.Context. Look for config in this context
instance.
context_key: str name of step config in context.
"""
assert name, ("name parameter must exist for a ControlOfFlowStep.")
assert context, ("context param must exist for ControlOfFlowStep.")
# this way, logs output as the calling step, which makes more sense
# to end-user than a mystery steps.dsl.blah logging output.
logger = logging.getLogger(name)
logger.debug("starting")
context.assert_key_has_value(key=context_key, caller=name)
original_config = (context_key, context[context_key])
config = context.get_formatted(context_key)
if isinstance(config, str):
groups = [config]
success_group = None
failure_group = None
elif isinstance(config, list):
groups = config
success_group = None
failure_group = None
elif isinstance(config, dict):
if 'groups' not in config:
raise KeyNotInContextError(
f"{context_key} needs a child key 'groups', which should "
"be a list or a str with the step-group name(s) you want "
f"to run. This is for step {name}.")
groups = config['groups']
if not groups:
raise KeyInContextHasNoValueError(
f"{context_key}.groups must have a value for step {name}")
if isinstance(groups, str):
groups = [groups]
success_group = config.get('success', None)
failure_group = config.get('failure', None)
else:
raise ContextError(
f"{context_key} needs a child key 'groups', which should "
"be a list or a str with the step-group name(s) you want "
f"to run. This is for step {name}. Instead, you've got {config}")
if success_group is not None and not isinstance(success_group, str):
raise ContextError(
f"{context_key}.success must be a string for {name}.")
if failure_group is not None and not isinstance(failure_group, str):
raise ContextError(
f"{context_key}.failure must be a string for {name}.")
logger.info(
("step %s about to hand over control with %s: Will run groups: %s "
" with success %s and failure %s"), name, context_key, groups,
success_group, failure_group)
raise instruction_type(groups=groups,
success_group=success_group,
failure_group=failure_group,
original_config=original_config)
def run_step(context):
"""Parse input string into Context as an object.
Args:
context: pypyr.context.Context. Mandatory.
The following context keys expected:
- jsonParse
- json. string or formatting expression evaluating to a
string of json.
- key. string. If exists, write json structure to this
context key. Else json writes to context root.
Returns:
None.
Raises:
pypyr.errors.KeyNotInContextError: jsonParse or jsonParse.json missing
in context.
pypyr.errors.KeyInContextHasNoValueError: jsonParse.json exists but is
empty.
"""
logger.debug("started")
context.assert_key_has_value('jsonParse', __name__)
input_context = context.get_formatted('jsonParse')
assert_key_exists(obj=input_context,
key='json',
caller=__name__,
parent='jsonParse')
destination_key = input_context.get('key', None)
json_string = input_context['json']
if not json_string:
raise KeyInContextHasNoValueError(
'jsonParse.json exists but is empty. It should be a valid json '
'string for pypyr.steps.jsonparse. For example: '
'\'{"key1": "value1", "key2": "value2"}\'')
payload = json.loads(json_string)
if destination_key:
logger.debug("json string parsed. Writing to context %s",
destination_key)
context[destination_key] = payload
else:
if not isinstance(payload, MutableMapping):
raise TypeError(
'json input should describe an object at the top '
'level when jsonParse.key isn\'t specified. You should have '
'something like \'{"key1": "value1", "key2": "value2"}\' '
'in the json top-level, not ["value1", "value2"]')
logger.debug("json string parsed. Merging into pypyr context at %s...",
destination_key)
context.update(payload)
logger.info("json string parsed into pypyr context.")
logger.debug("done")