def test_key_not_in_context_error_raises():
"""Key not in context error raises with correct message."""
# confirm subclassed from pypyr root error
assert isinstance(KeyNotInContextError(), PypyrError)
assert isinstance(KeyNotInContextError(), ContextError)
with pytest.raises(KeyNotInContextError) as err_info:
raise KeyNotInContextError("this is error text right here")
assert str(err_info.value) == "this is error text right here"
def get_waiter_args(context):
"""Gets required args from context for this step.
Args:
context - dict. context.
Returns:
tuple(client_in, service_name, waiter_name)
Raises:
pypyr.errors.KeyNotInContextError: Required key missing in context.
pypyr.errors.KeyInContextHasNoValueError: Required key exists but is
empty or None.
"""
try:
client_in = context['awsWaitIn']
service_name = client_in['serviceName']
waiter_name = client_in['waiterName']
except KeyError as err:
raise KeyNotInContextError(
"awsWaitIn missing required key for pypyraws.steps.wait: "
f"{err}"
) from err
# of course, if They went and made it a bool and True this will pass.
if not (service_name and service_name.strip()):
raise KeyInContextHasNoValueError(
'serviceName required in awsWaitIn for pypyraws.steps.wait')
if not (waiter_name and waiter_name.strip()):
raise KeyInContextHasNoValueError(
'waiterName required in awsWaitIn for pypyraws.steps.wait')
return client_in, service_name, waiter_name
def run_step(context):
"""Get, set, unset $ENVs.
Context is a dictionary or dictionary-like. context is mandatory.
Input context is:
env:
get: {dict}
set: {dict}
unset: [list]
At least one of env's sub-keys (get, set or unset) must exist.
This step will run whatever combination of Get, Set and Unset you specify.
Regardless of combination, execution order is Get, Set, Unset.
"""
logger.debug("started")
assert context, f"context must have value for {__name__}"
context.assert_key_has_value('env', __name__)
found_get = env_get(context)
found_set = env_set(context)
found_unset = env_unset(context)
# at least 1 of envGet, envSet or envUnset must exist in context
if not (found_get or found_set or found_unset):
raise KeyNotInContextError(
"context must contain any combination of "
"env['get'], env['set'] or env['unset'] for "
f"{__name__}")
logger.debug("done")
def __missing__(self, key):
"""Throw KeyNotInContextError rather than KeyError.
Python explicitly clears this over-ride for dict inheritance.
https://docs.python.org/3/library/stdtypes.html#dict
"""
raise KeyNotInContextError(f"{key} not found in the pypyr context.")
def get_awsclient_args(context, calling_module_name):
"""Get required args from context for awsClientIn type steps.
Args:
context: pypyr.context.Context.
calling_module_name: string. This is just to make a friendly error msg
should something go wrong.
Returns:
tuple(client_in, service_name, method_name)
Raises:
pypyr.errors.KeyNotInContextError: Required key missing in context.
pypyr.errors.KeyInContextHasNoValueError: Required key exists but is
empty or None.
"""
try:
client_in = context['awsClientIn']
service_name = client_in['serviceName']
method_name = client_in['methodName']
except KeyError as err:
raise KeyNotInContextError(
f"awsClientIn missing required key for {calling_module_name}: "
f"{err}") from err
if not (service_name and service_name.strip()):
raise KeyInContextHasNoValueError(
f'serviceName required in awsClientIn for {calling_module_name}')
if not (method_name and method_name.strip()):
raise KeyInContextHasNoValueError(
f'methodName required in awsClientIn for {calling_module_name}')
return client_in, service_name, method_name
def run_step(context):
"""Archive and/or extract tars with or without compression.
Args:
context: dictionary-like. Mandatory.
Expects the following context:
tar:
extract:
- in: /path/my.tar
out: /out/path
archive:
- in: /dir/to/archive
out: /out/destination.tar
format: ''
tar['format'] - if not specified, defaults to lzma/xz
Available options:
- '' - no compression
- gz (gzip)
- bz2 (bzip2)
- xz (lzma)
This step will run whatever combination of Extract and Archive you specify.
Regardless of combination, execution order is Extract, Archive.
Source and destination paths support {key} string interpolation.
Never extract archives from untrusted sources without prior inspection.
It is possible that files are created outside of path, e.g. members that
have absolute filenames starting with "/" or filenames with two dots "..".
"""
logger.debug("started")
assert context, f"context must have value for {__name__}"
found_at_least_one = False
context.assert_key_has_value('tar', __name__)
tar_context = context.get_formatted('tar')
if tar_context.get('extract', None):
found_at_least_one = True
tar_extract(tar_context)
if tar_context.get('archive', None):
found_at_least_one = True
tar_archive(tar_context)
if not found_at_least_one:
# This will raise exception on first item with a problem.
raise KeyNotInContextError('pypyr.steps.tar must have either extract '
'or archive specified under the tar key. '
'Or both of these. It has neither.')
logger.debug("done")
def run_step(context: Context):
print("filespec: %s" % str(context))
context.assert_key_has_value("vlnv", "filespec")
context.assert_key_exists("out", "filespec")
vlnv = context.get_formatted("vlnv")
dbm = CoreDbMgr.inst()
deps = dbm.get_depends(vlnv)
for e in context["out"]:
if "name" not in e.keys():
raise KeyNotInContextError("Missing 'name'")
if "type" not in e.keys():
raise KeyNotInContextError("Missing 'type'")
name = e["name"]
file_types = set()
for t in e["type"]:
file_types.add(t.strip())
flags = {}
if "flags" in e.keys():
for f in e["flags"]:
flags[f] = True
is_include = False
if "include" in e.keys():
is_include = bool(e["include"])
files = dbm.collect_files(deps, file_types, flags, is_include)
if name in context.keys():
if isinstance(context[name], list):
context[name].extend(files)
elif isinstance(context[name], str):
context[name] += " ".join(files)
else:
raise Exception("Target for files is an unsupported type %s" %
str(type(context[name])))
else:
context[name] = files
def get_args(get_item):
"""Parse env, key, default out of input dict.
Args:
get_item: dict. contains keys env/key/default
Returns:
(env, key, has_default, default) tuple, where
env: str. env var name.
key: str. save env value to this context key.
has_default: bool. True if default specified.
default: the value of default, if specified.
Raises:
ContextError: envGet is not a list of dicts.
KeyNotInContextError: If env or key not found in get_config.
"""
if not isinstance(get_item, dict):
raise ContextError('envGet must contain a list of dicts.')
env = get_item.get('env', None)
if not env:
raise KeyNotInContextError(
'context envGet[env] must exist in context for envGet.')
key = get_item.get('key', None)
if not key:
raise KeyNotInContextError(
'context envGet[key] must exist in context for envGet.')
if 'default' in get_item:
has_default = True
default = get_item['default']
else:
has_default = False
default = None
return (env, key, has_default, default)
def test_key_in_context_has_no_value_error_raises():
"""Key not in context value error raises with correct message."""
# confirm subclassed from pypyr root error
assert isinstance(KeyInContextHasNoValueError(), PypyrError)
assert isinstance(KeyNotInContextError(), ContextError)
with pytest.raises(KeyInContextHasNoValueError) as err_info:
raise KeyInContextHasNoValueError("this is error text right here")
assert repr(
err_info.value) == ("KeyInContextHasNoValueError('this is error "
"text right here',)")
def get_formatted(self, key):
"""Return formatted value for context[key].
If context[key] is a type string, will just format and return the
string.
If context[key] is a special literal type, like a py string or sic
string, will run the formatting implemented by the custom tag
representer.
If context[key] is not a string, specifically an iterable type like a
dict, list, tuple, set, it will use get_formatted_iterable under the
covers to loop through and handle the entire structure contained in
context[key].
Returns a string interpolated from the context dictionary.
If context[key]='Piping {key1} the {key2} wild'
And context={'key1': 'down', 'key2': 'valleys', 'key3': 'value3'}
Then this will return string: "Piping down the valleys wild"
get_formatted gets a context[key] value with formatting applied.
get_formatted_value is for any object.
get_formatted_string is for formatting any arbitrary string.
get_formatted_iterable. formats an input iterable.
Args:
key: dictionary key to retrieve.
Returns:
Formatted string.
Raises:
KeyNotInContextError: context[key] value contains {somekey} where
somekey does not exist in context dictionary.
"""
val = self[key]
if isinstance(val, str):
try:
return self.get_processed_string(val)
except KeyNotInContextError as err:
# Wrapping the KeyError into a less cryptic error for end-user
# friendliness
raise KeyNotInContextError(
f'Unable to format \'{val}\' at context[\'{key}\'], '
f'because {err}'
) from err
elif isinstance(val, SpecialTagDirective):
return val.get_value(self)
else:
# any sort of complex type will work with get_formatted_iterable.
return self.get_formatted_iterable(val)
def get_formatted(self, key):
"""Return formatted value for context[key].
This is a convenience method that calls the same thing as
get_formatted_value() under the hood, passing to it the value it
retrieves from context at the input key.
If context[key]'s value is a type string, will just format and return
the string. Strings can contain recursive formatting expressions.
If context[key]'s value is a special type, like a py string or sic
string, will run the formatting implemented by the custom tag
representer.
If context[key] is not a string, specifically an iterable type like a
dict, list, tuple, set, it will use get_formatted_value under the
covers to loop through and handle the entire structure contained in
context[key].
If context[key]='Piping {key1} the {key2} wild'
And context={'key1': 'down', 'key2': 'valleys', 'key3': 'value3'}
Then this will return string: "Piping down the valleys wild"
Choosing between get_formatted() and get_formatted_value():
- get_formatted() gets a context[key] value with formatting applied.
- get_formatted_value() is for any arbitrary object.
Args:
key: dictionary key to retrieve.
Returns:
Whatever object results from the formatting expression(s) at the
input key's value.
Raises:
KeyNotInContextError: context[key] value contains {somekey} where
somekey does not exist in context dictionary.
"""
val = self[key]
try:
# any sort of complex type will work with recursive formatter.
return self.formatter.vformat(val, None, self)
except KeyNotInContextError as err:
# less cryptic error for end-user friendliness
raise KeyNotInContextError(
f'Unable to format \'{val}\' at context[\'{key}\'], '
f'because {err}') from err
def assert_key_exists(self, key, caller):
"""Assert that context contains key.
Args:
key: validates that this key exists in context
caller: string. calling function or module name - this used to
construct error messages
Raises:
KeyNotInContextError: When key doesn't exist in context.
"""
assert key, ("key parameter must be specified.")
if key not in self:
raise KeyNotInContextError(
f"context['{key}'] doesn't exist. It must exist for {caller}.")
def get_payload(context):
"""Gets object from s3, reads underlying http stream, returns bytes.
Args:
context: pypyr.context.Context. Mandatory. Must contain key:
- s3Fetch: dict. mandatory. Must contain:
- methodArgs
- Bucket: string. s3 bucket name.
- Key: string. s3 key name.
Returns:
bytes: payload of the s3 obj in bytes
Raises:
KeyNotInContextError: s3Fetch or s3Fetch.methodArgs missing
"""
logger.debug("started")
assert context
fetch_me = context['s3Fetch']
try:
operation_args = fetch_me['methodArgs']
except KeyError as err:
raise KeyNotInContextError(
"s3Fetch missing required key for pypyraws.steps.s3fetch step: "
"methodArgs") from err
client_args = fetch_me.get('clientArgs', None)
if client_args is not None:
client_args = context.get_formatted_iterable(client_args)
operation_args = context.get_formatted_iterable(operation_args)
response = pypyraws.aws.service.operation_exec(
service_name='s3',
method_name='get_object',
client_args=client_args,
operation_args=operation_args)
logger.debug("reading response stream")
payload = response['Body']
logger.debug("returning response bytes")
logger.debug("done")
return payload
def get_formatted(self, key):
"""Returns formatted value for context[key].
If context[key] is a type string, will just format and return the
string.
If context[key] is not a string, specifically an iterable type like a
dict, list, tuple, set, it will use get_formatted_iterable under the
covers to loop through and handle the entire structure contained in
context[key].
Returns a string interpolated from the context dictionary.
If context[key]='Piping {key1} the {key2} wild'
And context={'key1': 'down', 'key2': 'valleys', 'key3': 'value3'}
Then this will return string: "Piping down the valleys wild"
Args:
key: dictionary key to retrieve.
Returns:
Formatted string.
Raises:
KeyNotInContextError: context[key] value contains {somekey} where
somekey does not exist in context dictionary.
"""
val = self[key]
if isinstance(val, str):
try:
return self.get_processed_string(val)
except KeyError as err:
# Wrapping the KeyError into a less cryptic error for end-user
# friendliness
missing_key = err.args[0]
raise KeyNotInContextError(
f'Unable to format \'{val}\' at context[\'{key}\'] with '
f'{{{missing_key}}}, because '
f'context[\'{missing_key}\'] doesn\'t exist'
) from err
else:
# any sort of complex type will work with get_formatted_iterable.
return self.get_formatted_iterable(val)
def get_formatted_string(self, input_string):
"""Use get_formatted_value(input_value) instead. Deprecated."""
from warnings import warn
warn(("Use get_formatted_value(input_value) instead of "
"get_formatted_string"), DeprecationWarning)
if isinstance(input_string, str):
try:
return self.formatter.vformat(input_string, None, self)
except KeyNotInContextError as err:
# Wrapping the KeyError into a less cryptic error for end-user
# friendliness
raise KeyNotInContextError(
f'Unable to format \'{input_string}\' because {err}'
) from err
elif isinstance(input_string, SpecialTagDirective):
return input_string.get_value(self)
else:
raise TypeError(f"can only format on strings. {input_string} is a "
f"{type(input_string)} instead.")
def get_formatted_string(self, input_string):
"""Return formatted value for input_string.
get_formatted gets a context[key] value with formatting applied.
get_formatted_value is for any object.
get_formatted_string is for formatting any arbitrary string.
get_formatted_iterable. formats an input iterable.
Only valid if input_string is a type string.
Return a string interpolated from the context dictionary.
If input_string='Piping {key1} the {key2} wild'
And context={'key1': 'down', 'key2': 'valleys', 'key3': 'value3'}
Then this will return string: "Piping down the valleys wild"
Args:
input_string: string to parse for substitutions.
Returns:
Formatted string.
Raises:
KeyNotInContextError: context[key] has {somekey} where somekey does
not exist in context dictionary.
TypeError: Attempt operation on a non-string type.
"""
if isinstance(input_string, str):
try:
return self.get_processed_string(input_string)
except KeyNotInContextError as err:
# Wrapping the KeyError into a less cryptic error for end-user
# friendliness
raise KeyNotInContextError(
f'Unable to format \'{input_string}\' because {err}'
) from err
elif isinstance(input_string, SpecialTagDirective):
return input_string.get_value(self)
else:
raise TypeError(f"can only format on strings. {input_string} is a "
f"{type(input_string)} instead.")
def assert_key_type_value(self,
context_item,
caller,
extra_error_text=''):
"""Assert that keys exist of right type and has a value.
Args:
context_item: ContextItemInfo tuple
caller: string. calling function name - this used to construct
error messages
extra_error_text: append to end of error message.
Raises:
AssertionError: if context_item None.
KeyNotInContextError: Key doesn't exist
KeyInContextHasNoValueError: context[key] is None or the wrong
type.
"""
assert context_item, ("context_item parameter must be specified.")
if extra_error_text is None or extra_error_text == '':
append_error_text = ''
else:
append_error_text = f' {extra_error_text}'
if not context_item.key_in_context:
raise KeyNotInContextError(f'{caller} couldn\'t find '
f'{context_item.key} in context.'
f'{append_error_text}')
if not context_item.has_value:
raise KeyInContextHasNoValueError(
f'{caller} found {context_item.key} in '
f'context but it doesn\'t have a value.'
f'{append_error_text}')
if not context_item.is_expected_type:
raise KeyInContextHasNoValueError(
f'{caller} found {context_item.key} in context, but it\'s '
f'not a {context_item.expected_type}.'
f'{append_error_text}')
def get_arguments(context):
"""Parse arguments for pype from context and assign default values.
Args:
context: pypyr.context.Context. context is mandatory.
Returns:
tuple (pipeline_name, #str
use_parent_context, #bool
pipe_arg, #str
skip_parse, #bool
raise_error #bool
)
Raises:
pypyr.errors.KeyNotInContextError: if ['pype']['name'] is missing.
pypyr.errors.KeyInContextHasNoValueError: if ['pype']['name'] exists but
is None.
"""
context.assert_key_has_value(key='pype', caller=__name__)
pype = context['pype']
try:
pipeline_name = pype['name']
if pipeline_name is None:
raise KeyInContextHasNoValueError(
"pypyr.steps.pype ['pype']['name'] exists but is empty.")
except KeyError as err:
raise KeyNotInContextError(
"pypyr.steps.pype missing 'name' in the 'pype' context item. "
"You need to specify the pipeline name to run another "
"pipeline.") from err
use_parent_context = pype.get('useParentContext', True)
pipe_arg = pype.get('pipeArg', None)
skip_parse = pype.get('skipParse', True)
raise_error = pype.get('raiseError', True)
return (pipeline_name, use_parent_context, pipe_arg, skip_parse,
raise_error)
def get_formatted_string(self, input_string):
"""Returns formatted value for input_string.
get_formatted gets a context[key] value.
get_formatted_string is for any arbitrary string that is not in the
context.
Only valid if input_string is a type string.
Return a string interpolated from the context dictionary.
If input_string='Piping {key1} the {key2} wild'
And context={'key1': 'down', 'key2': 'valleys', 'key3': 'value3'}
Then this will return string: "Piping down the valleys wild"
Args:
input_string: string to parse for substitutions.
Returns:
Formatted string.
Raises:
KeyError: context[key] has {somekey} where somekey does not exist
in context dictionary.
TypeError: Attempt operation on a non-string type.
"""
if isinstance(input_string, str):
try:
return self.get_processed_string(input_string)
except KeyError as err:
# Wrapping the KeyError into a less cryptic error for end-user
# friendliness
missing_key = err.args[0]
raise KeyNotInContextError(
f'Unable to format \'{input_string}\' with '
f'{{{missing_key}}}, because '
f'context[\'{missing_key}\'] doesn\'t exist') from err
else:
raise TypeError(f"can only format on strings. {input_string} is a "
f"{type(input_string)} instead.")
def get_awsclient_args(input_dict, calling_module_name):
"""Get required args from context for awsClientIn type steps.
Doesn't do any formatting. You gotta format before you get here.
Args:
input_dict (dict): dict-like structure containing awsClientIn key.
calling_module_name: string. This is just to make a friendly error msg
should something go wrong.
Returns:
tuple(service_name, method_name, client_args, method_args)
Raises:
pypyr.errors.KeyNotInContextError: Required key missing in context.
pypyr.errors.KeyInContextHasNoValueError: Required key exists but is
empty or None.
"""
try:
service_name = input_dict['serviceName']
method_name = input_dict['methodName']
except KeyError as err:
raise KeyNotInContextError(
f"awsClientIn missing required key for {calling_module_name}: "
f"{err}"
) from err
if not (service_name and service_name.strip()):
raise KeyInContextHasNoValueError(
f'serviceName required in awsClientIn for {calling_module_name}')
if not (method_name and method_name.strip()):
raise KeyInContextHasNoValueError(
f'methodName required in awsClientIn for {calling_module_name}')
client_args = input_dict.get('clientArgs', None)
method_args = input_dict.get('methodArgs', None)
return service_name, method_name, client_args, method_args
def assert_key_exists(obj, key, caller, parent=None):
"""Assert that object contains key.
Error messages are structured as if obj is a pypyr Context.
Args:
obj (mapping): object to check for key.
key (any valid key type): validates that this key exists in context
caller: string. calling function or module name - this used to
construct error messages. Tip: use .__name__
parent (any valid key type): parent key name. Used to construct error
messages to indicate the name of missing
obj in context.
Raises:
KeyNotInContextError: When key doesn't exist in context.
"""
try:
if key not in obj:
if parent:
msg = (f"context[{parent!r}][{key!r}] doesn't "
f"exist. It must exist for {caller}.")
else:
msg = (f"context[{key!r}] doesn't exist. "
f"It must exist for {caller}.")
raise KeyNotInContextError(msg)
except TypeError as err:
# catches None on obj or obj not iterable
if parent:
msg = (f"context[{parent!r}] must exist, be iterable and contain "
f"{key!r} for {caller}. {err}")
else:
msg = (f"context[{key!r}] must exist and be iterable for "
f"{caller}. {err}")
raise ContextError(msg) from err
def assert_child_key_has_value(self, parent, child, caller):
"""Assert that context contains key that has child which has a value.
Args:
parent: parent key
child: validate this sub-key of parent exists AND isn't None.
caller: string. calling function name - this used to construct
error messages
Raises:
KeyNotInContextError: Key doesn't exist
KeyInContextHasNoValueError: context[key] is None
AssertionError: if key is None
"""
assert parent, ("parent parameter must be specified.")
assert child, ("child parameter must be specified.")
self.assert_key_has_value(parent, caller)
try:
child_exists = child in self[parent]
except TypeError as err:
# This happens if parent isn't iterable
raise ContextError(
f"context['{parent}'] must be iterable and contain '{child}' "
f"for {caller}. {err}") from err
if child_exists:
if self[parent][child] is None:
raise KeyInContextHasNoValueError(
f"context['{parent}']['{child}'] must have a value for "
f"{caller}.")
else:
raise KeyNotInContextError(
f"context['{parent}']['{child}'] doesn't "
f"exist. It must exist for {caller}.")
def run_step(context):
"""Assert that something is True or equal to something else.
Args:
context (pypyr.context.Context): context is mandatory.
Uses the following context keys in context:
- assert
- this (any): mandatory. If assert['equals'] not specified,
eval as boolean.
- equals (any): optional. Any type.
Takes one of three input forms:
assert: evaluate me
or
assert:
this: evaluate me
or
assert:
this: compare me
equals: to this
If context['assert'] is not a dict, evaluate contents as bool.
If context['assert'] is a dict:
- If context['assert']['this'] evaluates to False raises error.
- If context['assert']['equals'] exists, raises error if assert.this
!= assert.equals.
All input forms support string substitutions.
Returns:
None
Raises:
AssertionError: if assert evaluates to False.
"""
logger.debug("started")
assert context, f"context must have value for {__name__}"
context.assert_key_exists('assert', __name__)
assert_context = context.get_formatted('assert')
is_equals_there = False
is_this_there = False
if isinstance(assert_context, dict):
is_this_there = 'this' in assert_context
if is_this_there:
assert_this = assert_context['this']
else:
assert_this = assert_context
is_equals_there = 'equals' in assert_context
if is_equals_there:
if not is_this_there:
raise KeyNotInContextError(
"you have to set assert.this to use assert.equals.")
assert_equals = assert_context['equals']
# compare assertThis to assertEquals
logger.debug("comparing assert['this'] to assert['equals'].")
assert_result = (assert_this == assert_equals)
else:
# nothing to compare means treat assert or assert.this as a bool.
if is_this_there:
logger.debug("evaluating assert['this'] as a boolean.")
else:
logger.debug("assert is a dict but contains no `this`. "
"evaluating assert value as a boolean.")
assert_result = cast_to_bool(assert_this)
else:
# assert key has a non-dict value so eval directly as a bool.
logger.debug("evaluating assert value as a boolean.")
assert_result = cast_to_bool(assert_context)
logger.info("assert evaluated to %s", assert_result)
if not assert_result:
if is_equals_there:
# emit type to help user, but not the actual field contents.
type_this = type(assert_this).__name__
type_equals = type(assert_equals).__name__
error_text = (
f"assert assert['this'] is of type {type_this} "
f"and does not equal assert['equals'] of type {type_equals}.")
else:
og_assert = (context['assert']['this']
if is_this_there else context['assert'])
# original literal hard-coded in pipe, so presumably not a
# sensitive value.
error_text = (
f"assert {og_assert} evaluated to False.")
raise AssertionError(error_text)
logger.debug("done")
def run_step(context):
"""Run me after an ecs task run or stop to prepare an ecs waiter.
Prepares the awsWaitIn context key for pypyraws.steps.wait
Use this step after any of the following methods if you want to use one of
the ecs waiters to wait for a specific state:
- describe_services
- describe_tasks
- list_services - specify awsEcsWaitPrepCluster if you don't want default
- list_tasks - specify awsEcsWaitPrepCluster if you don't want default
- run_task
- start_task
- stop_task
- update_service
You don't have to use this step, you could always just construct the
awsWaitIn dictionary in context yourself. It just so happens this step
saves you some legwork to do so.
Args:
context:
Dictionary. Mandatory.
Requires the following context keys in context:
- awsClientOut. dict. mandatory. This is the context key that
any ecs command executed by pypyraws.steps.service adds.
Chances are pretty good you don't want to construct this by
hand yourself - the idea is to use the output as generated by
one of the supported ecs methods.
- awsEcsWaitPrepCluster. string. optional. The short name or
full arn of the cluster that hosts the task to describe. If
you do not specify a cluster, the default cluster is assumed.
For most of the ecs methods the code automatically deduces
the cluster from awsClientOut, so don't worry about it.
But, when following list_services and list_tasks, you have to
specify this parameter. Specifying this parameter will
override any automatically deduced cluster arn.
Returns:
None.
Overwrites the awsWaitIn key in context. The new awsWaitIn will contain
waitArgs filled with the task or service arns found in awsClientOut.
Raises:
pypyr.errors.KeyNotInContextError: awsClientOut missing in context.
pypyr.errors.KeyInContextHasNoValueError: awsClientOut exists but is
None.
"""
logger.debug("started")
context.assert_key_has_value('awsClientOut', __name__)
if 'awsEcsWaitPrepCluster' in context:
logger.debug("awsEcsWaitPrepCluster specified in input context")
# awsEcsWaitPrepCluster always overrides automatically deduced cluster
cluster = context['awsEcsWaitPrepCluster']
else:
cluster = None
parse_me = context['awsClientOut']
isTask = False
isService = False
if 'service' in parse_me:
logger.debug("Found 'service' in awsClientOut")
if cluster is None:
cluster = parse_me['service']['clusterArn']
arn_list = [parse_me['service']['serviceArn']]
isService = True
elif 'serviceArns' in parse_me:
logger.debug("Found 'serviceArns' in awsClientOut")
# no cluster arn is available for these
arn_list = parse_me['serviceArns']
isService = True
elif 'services' in parse_me:
logger.debug("Found 'services' in awsClientOut")
if cluster is None:
# so happens aws methods returning 'services' all do so with only 1
# cluster input.
cluster = parse_me['services'][0]['clusterArn']
arn_list = [svc['serviceArn'] for svc in parse_me['services']]
isService = True
elif 'task' in parse_me:
logger.debug("Found 'task' in awsClientOut")
if cluster is None:
cluster = parse_me['task']['clusterArn']
arn_list = [parse_me['task']['taskArn']]
isTask = True
elif 'taskArns' in parse_me:
logger.debug("Found 'taskArns' in awsClientOut")
# no cluster arn is available for these
arn_list = parse_me['taskArns']
isTask = True
elif 'tasks' in parse_me:
logger.debug("Found 'tasks' in awsClientOut")
if cluster is None:
# so happens the aws methods that returns 'tasks' all do so with
# only 1 cluster input, i.e cluster will be the same for all of the
# return tasks
cluster = parse_me['tasks'][0]['clusterArn']
arn_list = [task['taskArn'] for task in parse_me['tasks']]
isTask = True
else:
raise KeyNotInContextError('Run ecswaitprep after an ecs method that '
'does something with services or tasks. '
'Couldn\'t find service, serviceArns, '
'services, task, taskArns or tasks in '
'awsClientOut.')
waiter_dict = {}
if cluster is None:
logger.debug("No cluster specified. Waiter will use default cluster.")
else:
logger.debug(f"{cluster} cluster specified.")
waiter_dict['cluster'] = cluster
if isTask:
logger.debug("Adding task arns")
waiter_dict['tasks'] = arn_list
if isService:
logger.debug("Adding service arns")
waiter_dict['services'] = arn_list
if 'awsWaitIn' not in context:
context['awsWaitIn'] = {}
context['awsWaitIn']['waitArgs'] = waiter_dict
logger.info("added context['awsWaitIn']['waitArgs']")
logger.debug("done")
def control_of_flow_instruction(name, instruction_type, context, context_key):
"""Run a control of flow instruction.
The step config in the context dict looks like this:
<<instruction-name>>: <<cmd string>>. Mandatory.
OR, as a dict
<<instruction-name:
groups: <<str>> or <<list of str>> - mandatory.
success: <<str>>
failure: <<str>>
Args:
name: Unique name for step. Likely __name__ of calling step.
instruction_type: Type - must inherit from
pypyr.errors.ControlOfFlowInstruction
context: pypyr.context.Context. Look for config in this context
instance.
context_key: str name of step config in context.
"""
assert name, ("name parameter must exist for a ControlOfFlowStep.")
assert context, ("context param must exist for ControlOfFlowStep.")
# this way, logs output as the calling step, which makes more sense
# to end-user than a mystery steps.dsl.blah logging output.
logger = logging.getLogger(name)
logger.debug("starting")
context.assert_key_has_value(key=context_key, caller=name)
original_config = (context_key, context[context_key])
config = context.get_formatted(context_key)
if isinstance(config, str):
groups = [config]
success_group = None
failure_group = None
elif isinstance(config, list):
groups = config
success_group = None
failure_group = None
elif isinstance(config, dict):
if 'groups' not in config:
raise KeyNotInContextError(
f"{context_key} needs a child key 'groups', which should "
"be a list or a str with the step-group name(s) you want "
f"to run. This is for step {name}.")
groups = config['groups']
if not groups:
raise KeyInContextHasNoValueError(
f"{context_key}.groups must have a value for step {name}")
if isinstance(groups, str):
groups = [groups]
success_group = config.get('success', None)
failure_group = config.get('failure', None)
else:
raise ContextError(
f"{context_key} needs a child key 'groups', which should "
"be a list or a str with the step-group name(s) you want "
f"to run. This is for step {name}. Instead, you've got {config}")
if success_group is not None and not isinstance(success_group, str):
raise ContextError(
f"{context_key}.success must be a string for {name}.")
if failure_group is not None and not isinstance(failure_group, str):
raise ContextError(
f"{context_key}.failure must be a string for {name}.")
logger.info(
("step %s about to hand over control with %s: Will run groups: %s "
" with success %s and failure %s"), name, context_key, groups,
success_group, failure_group)
raise instruction_type(groups=groups,
success_group=success_group,
failure_group=failure_group,
original_config=original_config)
def get_arguments(context):
"""Parse arguments for pype from context and assign default values.
Args:
context: pypyr.context.Context. context is mandatory.
Returns:
tuple (pipeline_name, #str
args, #dict
out, #str or dict or list
use_parent_context, #bool
pipe_arg, #str
skip_parse, #bool
raise_error #bool
groups #list of str
success_group #str
failure_group #str
)
Raises:
pypyr.errors.KeyNotInContextError: if ['pype']['name'] is missing.
pypyr.errors.KeyInContextHasNoValueError: if ['pype']['name'] exists but
is None.
"""
context.assert_key_has_value(key='pype', caller=__name__)
pype = context.get_formatted('pype')
try:
pipeline_name = pype['name']
if pipeline_name is None:
raise KeyInContextHasNoValueError(
"pypyr.steps.pype ['pype']['name'] exists but is empty.")
except KeyError as err:
raise KeyNotInContextError(
"pypyr.steps.pype missing 'name' in the 'pype' context item. "
"You need to specify the pipeline name to run another "
"pipeline.") from err
args = pype.get('args', None)
if args is not None and not isinstance(args, dict):
raise ContextError(
"pypyr.steps.pype 'args' in the 'pype' context item "
"must be a dict.")
if args and 'useParentContext' not in pype:
use_parent_context = False
else:
use_parent_context = pype.get('useParentContext', True)
out = pype.get('out', None)
if out and use_parent_context:
raise ContextError(
"pypyr.steps.pype pype.out is only relevant if useParentContext "
"= False. If you're using the parent context, no need to have out "
"args since their values will already be in context. If you're "
"NOT using parent context and you've specified pype.args, just "
"leave off the useParentContext key and it'll default to False "
"under the hood, or set it to False yourself if you keep it in.")
pipe_arg = pype.get('pipeArg', None)
skip_parse = pype.get('skipParse', True)
raise_error = pype.get('raiseError', True)
loader = pype.get('loader', None)
groups = pype.get('groups', None)
if isinstance(groups, str):
groups = [groups]
success_group = pype.get('success', None)
failure_group = pype.get('failure', None)
return (pipeline_name, args, out, use_parent_context, pipe_arg, skip_parse,
raise_error, loader, groups, success_group, failure_group)
