def test_assert_key_has_value_none_no_parent():
"""Key value is none and no parent."""
with pytest.raises(KeyInContextHasNoValueError) as err:
asserts.assert_key_has_value(obj={'k1': None},
key='k1',
caller='arb caller')
assert str(err.value) == "context['k1'] must have a value for arb caller."
def test_assert_key_has_value_key_not_there_no_parent():
"""Key not in object and no parent."""
with pytest.raises(KeyNotInContextError) as err:
asserts.assert_key_has_value(obj={'k1': None},
key='k2',
caller='arb caller')
assert str(err.value) == ("context['k2'] doesn't exist. "
"It must exist for arb caller.")
def test_assert_key_has_value_none():
"""Key value is none."""
with pytest.raises(KeyInContextHasNoValueError) as err:
asserts.assert_key_has_value(obj={'k1': None},
key='k1',
caller='arb caller',
parent='parent name')
assert str(err.value) == ("context['parent name']['k1'] must have a value "
"for arb caller.")
def test_assert_key_has_value_object_not_iterable_no_parent():
"""Object is None should raise with no parent."""
with pytest.raises(ContextError) as err:
asserts.assert_key_has_value(obj=1,
key=2,
caller='arb caller')
assert str(err.value) == (
"context[2] must exist and be iterable for arb caller. argument of "
"type 'int' is not iterable")
def test_assert_key_has_value_object_not_iterable():
"""Object is None should raise."""
with pytest.raises(ContextError) as err:
asserts.assert_key_has_value(obj=1,
key=2,
caller='arb caller',
parent=3)
assert str(err.value) == (
"context[3] must exist, be iterable and contain 2 for "
"arb caller. argument of type 'int' is not iterable")
def test_assert_key_has_value_object_none():
"""Object is None should raise."""
with pytest.raises(ContextError) as err:
asserts.assert_key_has_value(obj=None,
key='k1',
caller='arb caller',
parent='parent name')
assert str(err.value) == (
"context['parent name'] must exist, be iterable and contain 'k1' for "
"arb caller. argument of type 'NoneType' is not iterable")
def run_step(context):
"""Write payload out to yaml file.
Args:
context: pypyr.context.Context. Mandatory.
The following context keys expected:
- fileWriteYaml
- path. mandatory. path-like. Write output file to
here. Will create directories in path for you.
- payload. optional. Write this to output file. If not
specified, output entire context.
Returns:
None.
Raises:
pypyr.errors.KeyNotInContextError: fileWriteYaml or
fileWriteYaml['path'] missing in context.
pypyr.errors.KeyInContextHasNoValueError: fileWriteYaml or
fileWriteYaml['path'] exists but is None.
"""
logger.debug("started")
context.assert_key_has_value('fileWriteYaml', __name__)
input_context = context.get_formatted('fileWriteYaml')
assert_key_has_value(obj=input_context,
key='path',
caller=__name__,
parent='fileWriteYaml')
out_path = input_context['path']
# doing it like this to safeguard against accidentally dumping all context
# with potentially sensitive values in it to disk if payload exists but is
# None.
is_payload_specified = 'payload' in input_context
yaml_writer = pypyr.yaml.get_yaml_parser_roundtrip_for_context()
logger.debug("opening destination file for writing: %s", out_path)
os.makedirs(os.path.abspath(os.path.dirname(out_path)), exist_ok=True)
with open(out_path, 'w') as outfile:
if is_payload_specified:
payload = input_context['payload']
else:
payload = context.get_formatted_value(context)
yaml_writer.dump(payload, outfile)
logger.info("formatted context content and wrote to %s", out_path)
logger.debug("done")
def assert_key_has_value(self, key, caller):
"""Assert that context contains key which also has a value.
Args:
key: validate this key exists in context AND has a value that isn't
None.
caller: string. calling function name - this used to construct
error messages
Raises:
KeyNotInContextError: Key doesn't exist
KeyInContextHasNoValueError: context[key] is None
AssertionError: if key is None
"""
asserts.assert_key_has_value(self, key, caller)
def assert_child_key_has_value(self, parent, child, caller):
"""Assert that context contains key that has child which has a value.
Args:
parent: parent key
child: validate this sub-key of parent exists AND isn't None.
caller: string. calling function name - this used to construct
error messages
Raises:
KeyNotInContextError: Key doesn't exist
KeyInContextHasNoValueError: context[key] is None
AssertionError: if key is None
"""
asserts.assert_key_has_value(self, parent, caller)
asserts.assert_key_has_value(self[parent], child, caller, parent)
def run_step(context):
"""Load a yaml file into the pypyr context.
Yaml parsed from the file will be merged into the pypyr context. This will
overwrite existing values if the same keys are already in there.
I.e if file yaml has {'eggs' : 'boiled'} and context {'eggs': 'fried'}
already exists, returned context['eggs'] will be 'boiled'.
Args:
context: pypyr.context.Context. Mandatory.
The following context key must exist
- fetchYaml
- path. path-like. Path to file on disk.
- key. string. If exists, write yaml to this context key.
Else yaml writes to context root.
All inputs support formatting expressions.
Also supports a passing path as string to fetchYaml, but in this case you
won't be able to specify a key.
Returns:
None. updates context arg.
Raises:
FileNotFoundError: take a guess
pypyr.errors.KeyNotInContextError: fetchYamlPath missing in context.
pypyr.errors.KeyInContextHasNoValueError: fetchYamlPath exists but is
None.
"""
logger.debug("started")
context.assert_key_has_value(key='fetchYaml', caller=__name__)
fetch_yaml_input = context.get_formatted('fetchYaml')
if isinstance(fetch_yaml_input, str):
file_path = fetch_yaml_input
destination_key = None
else:
assert_key_has_value(obj=fetch_yaml_input,
key='path',
caller=__name__,
parent='fetchYaml')
file_path = fetch_yaml_input['path']
destination_key = fetch_yaml_input.get('key', None)
logger.debug("attempting to open file: %s", file_path)
with open(file_path) as yaml_file:
yaml_loader = yaml.YAML(typ='safe', pure=True)
payload = yaml_loader.load(yaml_file)
if destination_key:
logger.debug("yaml file loaded. Writing to context %s",
destination_key)
context[destination_key] = payload
else:
if not isinstance(payload, MutableMapping):
raise TypeError(
"yaml input should describe a dictionary at the top "
"level when fetchYamlKey isn't specified. You should have "
"something like \n'key1: value1'\n key2: value2'\n"
"in the yaml top-level, not \n'- value1\n - value2'")
logger.debug("yaml file loaded. Merging into pypyr context. . .")
context.update(payload)
logger.info("yaml file written into pypyr context. Count: %s",
len(payload))
logger.debug("done")
def run_step(context):
"""Wait for any aws client operation.
All of the awsWaitFor descendant values support {key}
string interpolation, except waitForField.
Args:
context:
Dictionary. Mandatory.
Requires the following context keys in context:
- awsWaitFor. dict. mandatory. Contains keys:
- awsClientIn. dict. mandatory. This is the same as for the
pypyraws.steps.client in step. Contains keys:
- serviceName: mandatory. String for service name.
Available services here:
http://boto3.readthedocs.io/en/latest/reference/services/
- methodName: mandatory. String. Name of method to
execute.
- clientArgs: optional. Dict. kwargs for the boto client
ctor.
- methodArgs: optional. Dict. kwargs for the client
method call
- waitForField: mandatory. string. format expression for
field name to check in awsClient response.
- toBe: mandatory. string. string. Stop waiting when
waitForField equals this value.
- pollInterval: optional. int. In seconds. Default to 30.
- maxAttempts: optional. int. Default 10.
- errorOnWaitTimeout: optional. Default True. Throws error
if maxAttempts exhausted without
reaching toBe value. If false,
step completes without raising
error.
Returns: None
Adds key to context:
- awsWaitForTimedOut: bool. Adds key with value True if
errorOnWaitTimeout=False and max_attempts exhausted without
reaching toBe. If steps completes successfully and waitForField's
value becomes toBe, awsWaitForTimedOut == False.
Raises:
pypyr.errors.KeyNotInContextError: awsWaitFor missing in context.
pypyr.errors.KeyInContextHasNoValueError: awsWaitFor exists but is
None.
pypyraws.errors.WaitTimeOut: maxAttempts exceeded without waitForField
changing to toBe.
"""
logger.debug("started")
context.assert_key_has_value('awsWaitFor', __name__)
wait_for = context['awsWaitFor']
assert_key_has_value(wait_for, 'awsClientIn', __name__, 'awsWaitFor')
aws_client_in = context.get_formatted_value(wait_for['awsClientIn'])
(service_name, method_name, client_args,
method_args) = contextargs.get_awsclient_args(aws_client_in, __name__)
(wait_for_field, to_be, poll_interval, max_attempts,
error_on_wait_timeout) = get_poll_args(wait_for, context)
wait_response = wait_until_true(
interval=poll_interval,
max_attempts=max_attempts)(execute_aws_client_method)(
service_name=service_name,
method_name=method_name,
client_args=client_args,
method_args=method_args,
wait_for_field=wait_for_field,
to_be=to_be)
if wait_response:
context['awsWaitForTimedOut'] = False
logger.info(f"aws {service_name} {method_name} returned {to_be}. "
"Pipeline will now continue.")
else:
if error_on_wait_timeout:
context['awsWaitForTimedOut'] = True
logger.error(f"aws {service_name} {method_name} did not return "
f"{to_be} within {max_attempts}. errorOnWaitTimeout "
"is True, throwing error")
raise WaitTimeOut(f"aws {service_name} {method_name} did not "
f"return {to_be} within {max_attempts} retries.")
else:
context['awsWaitForTimedOut'] = True
logger.warning(
f"aws {service_name} {method_name} did NOT return "
f" {to_be}. errorOnWaitTimeout is False, so pipeline "
"will proceed to the next step anyway.")
logger.debug("done")
def test_assert_key_value_empty():
"""Not a truthy."""
asserts.assert_key_has_value(obj={'k1': ''},
key='k1',
caller='arb caller',
parent='parent name')
def test_assert_key_has_value_passes():
"""Success case."""
asserts.assert_key_has_value(obj={'k1': 'v1'},
key='k1',
caller='arb caller',
parent='parent name')
def run_step(context):
"""Load a json file into the pypyr context.
json parsed from the file will be merged into the pypyr context. This will
overwrite existing values if the same keys are already in there.
I.e if file json has {'eggs' : 'boiled'} and context {'eggs': 'fried'}
already exists, returned context['eggs'] will be 'boiled'.
The json should not be an array [] on the top level, but rather an Object.
Args:
context: pypyr.context.Context. Mandatory.
The following context key must exist
- fetchJson
- path. path-like. Path to file on disk.
- key. string. If exists, write json structure to this
context key. Else json writes to context root.
Also supports a passing path as string to fetchJson, but in this case you
won't be able to specify a key.
All inputs support formatting expressions.
Returns:
None. updates context arg.
Raises:
FileNotFoundError: take a guess
pypyr.errors.KeyNotInContextError: fetchJson.path missing in context.
pypyr.errors.KeyInContextHasNoValueError: fetchJson.path exists but is
None.
"""
logger.debug("started")
context.assert_key_has_value(key='fetchJson', caller=__name__)
fetch_json_input = context.get_formatted('fetchJson')
if isinstance(fetch_json_input, str):
file_path = fetch_json_input
destination_key = None
else:
assert_key_has_value(obj=fetch_json_input,
key='path',
caller=__name__,
parent='fetchJson')
file_path = fetch_json_input['path']
destination_key = fetch_json_input.get('key', None)
logger.debug("attempting to open file: %s", file_path)
with open(file_path) as json_file:
payload = json.load(json_file)
if destination_key:
logger.debug("json file loaded. Writing to context %s",
destination_key)
context[destination_key] = payload
else:
if not isinstance(payload, MutableMapping):
raise TypeError(
'json input should describe an object at the top '
'level when fetchJsonKey isn\'t specified. You should have '
'something like {"key1": "value1", "key2": "value2"} '
'in the json top-level, not ["value1", "value2"]')
logger.debug("json file loaded. Merging into pypyr context. . .")
context.update(payload)
logger.info("json file written into pypyr context. Count: %s",
len(payload))
logger.debug("done")