def test_assert_key_has_value_none_no_parent(): """Key value is none and no parent.""" with pytest.raises(KeyInContextHasNoValueError) as err: asserts.assert_key_has_value(obj={'k1': None}, key='k1', caller='arb caller') assert str(err.value) == "context['k1'] must have a value for arb caller."
def test_assert_key_has_value_key_not_there_no_parent(): """Key not in object and no parent.""" with pytest.raises(KeyNotInContextError) as err: asserts.assert_key_has_value(obj={'k1': None}, key='k2', caller='arb caller') assert str(err.value) == ("context['k2'] doesn't exist. " "It must exist for arb caller.")
def test_assert_key_has_value_none(): """Key value is none.""" with pytest.raises(KeyInContextHasNoValueError) as err: asserts.assert_key_has_value(obj={'k1': None}, key='k1', caller='arb caller', parent='parent name') assert str(err.value) == ("context['parent name']['k1'] must have a value " "for arb caller.")
def test_assert_key_has_value_object_not_iterable_no_parent(): """Object is None should raise with no parent.""" with pytest.raises(ContextError) as err: asserts.assert_key_has_value(obj=1, key=2, caller='arb caller') assert str(err.value) == ( "context[2] must exist and be iterable for arb caller. argument of " "type 'int' is not iterable")
def test_assert_key_has_value_object_not_iterable(): """Object is None should raise.""" with pytest.raises(ContextError) as err: asserts.assert_key_has_value(obj=1, key=2, caller='arb caller', parent=3) assert str(err.value) == ( "context[3] must exist, be iterable and contain 2 for " "arb caller. argument of type 'int' is not iterable")
def test_assert_key_has_value_object_none(): """Object is None should raise.""" with pytest.raises(ContextError) as err: asserts.assert_key_has_value(obj=None, key='k1', caller='arb caller', parent='parent name') assert str(err.value) == ( "context['parent name'] must exist, be iterable and contain 'k1' for " "arb caller. argument of type 'NoneType' is not iterable")
def run_step(context): """Write payload out to yaml file. Args: context: pypyr.context.Context. Mandatory. The following context keys expected: - fileWriteYaml - path. mandatory. path-like. Write output file to here. Will create directories in path for you. - payload. optional. Write this to output file. If not specified, output entire context. Returns: None. Raises: pypyr.errors.KeyNotInContextError: fileWriteYaml or fileWriteYaml['path'] missing in context. pypyr.errors.KeyInContextHasNoValueError: fileWriteYaml or fileWriteYaml['path'] exists but is None. """ logger.debug("started") context.assert_key_has_value('fileWriteYaml', __name__) input_context = context.get_formatted('fileWriteYaml') assert_key_has_value(obj=input_context, key='path', caller=__name__, parent='fileWriteYaml') out_path = input_context['path'] # doing it like this to safeguard against accidentally dumping all context # with potentially sensitive values in it to disk if payload exists but is # None. is_payload_specified = 'payload' in input_context yaml_writer = pypyr.yaml.get_yaml_parser_roundtrip_for_context() logger.debug("opening destination file for writing: %s", out_path) os.makedirs(os.path.abspath(os.path.dirname(out_path)), exist_ok=True) with open(out_path, 'w') as outfile: if is_payload_specified: payload = input_context['payload'] else: payload = context.get_formatted_value(context) yaml_writer.dump(payload, outfile) logger.info("formatted context content and wrote to %s", out_path) logger.debug("done")
def assert_key_has_value(self, key, caller): """Assert that context contains key which also has a value. Args: key: validate this key exists in context AND has a value that isn't None. caller: string. calling function name - this used to construct error messages Raises: KeyNotInContextError: Key doesn't exist KeyInContextHasNoValueError: context[key] is None AssertionError: if key is None """ asserts.assert_key_has_value(self, key, caller)
def assert_child_key_has_value(self, parent, child, caller): """Assert that context contains key that has child which has a value. Args: parent: parent key child: validate this sub-key of parent exists AND isn't None. caller: string. calling function name - this used to construct error messages Raises: KeyNotInContextError: Key doesn't exist KeyInContextHasNoValueError: context[key] is None AssertionError: if key is None """ asserts.assert_key_has_value(self, parent, caller) asserts.assert_key_has_value(self[parent], child, caller, parent)
def run_step(context): """Load a yaml file into the pypyr context. Yaml parsed from the file will be merged into the pypyr context. This will overwrite existing values if the same keys are already in there. I.e if file yaml has {'eggs' : 'boiled'} and context {'eggs': 'fried'} already exists, returned context['eggs'] will be 'boiled'. Args: context: pypyr.context.Context. Mandatory. The following context key must exist - fetchYaml - path. path-like. Path to file on disk. - key. string. If exists, write yaml to this context key. Else yaml writes to context root. All inputs support formatting expressions. Also supports a passing path as string to fetchYaml, but in this case you won't be able to specify a key. Returns: None. updates context arg. Raises: FileNotFoundError: take a guess pypyr.errors.KeyNotInContextError: fetchYamlPath missing in context. pypyr.errors.KeyInContextHasNoValueError: fetchYamlPath exists but is None. """ logger.debug("started") context.assert_key_has_value(key='fetchYaml', caller=__name__) fetch_yaml_input = context.get_formatted('fetchYaml') if isinstance(fetch_yaml_input, str): file_path = fetch_yaml_input destination_key = None else: assert_key_has_value(obj=fetch_yaml_input, key='path', caller=__name__, parent='fetchYaml') file_path = fetch_yaml_input['path'] destination_key = fetch_yaml_input.get('key', None) logger.debug("attempting to open file: %s", file_path) with open(file_path) as yaml_file: yaml_loader = yaml.YAML(typ='safe', pure=True) payload = yaml_loader.load(yaml_file) if destination_key: logger.debug("yaml file loaded. Writing to context %s", destination_key) context[destination_key] = payload else: if not isinstance(payload, MutableMapping): raise TypeError( "yaml input should describe a dictionary at the top " "level when fetchYamlKey isn't specified. You should have " "something like \n'key1: value1'\n key2: value2'\n" "in the yaml top-level, not \n'- value1\n - value2'") logger.debug("yaml file loaded. Merging into pypyr context. . .") context.update(payload) logger.info("yaml file written into pypyr context. Count: %s", len(payload)) logger.debug("done")
def run_step(context): """Wait for any aws client operation. All of the awsWaitFor descendant values support {key} string interpolation, except waitForField. Args: context: Dictionary. Mandatory. Requires the following context keys in context: - awsWaitFor. dict. mandatory. Contains keys: - awsClientIn. dict. mandatory. This is the same as for the pypyraws.steps.client in step. Contains keys: - serviceName: mandatory. String for service name. Available services here: http://boto3.readthedocs.io/en/latest/reference/services/ - methodName: mandatory. String. Name of method to execute. - clientArgs: optional. Dict. kwargs for the boto client ctor. - methodArgs: optional. Dict. kwargs for the client method call - waitForField: mandatory. string. format expression for field name to check in awsClient response. - toBe: mandatory. string. string. Stop waiting when waitForField equals this value. - pollInterval: optional. int. In seconds. Default to 30. - maxAttempts: optional. int. Default 10. - errorOnWaitTimeout: optional. Default True. Throws error if maxAttempts exhausted without reaching toBe value. If false, step completes without raising error. Returns: None Adds key to context: - awsWaitForTimedOut: bool. Adds key with value True if errorOnWaitTimeout=False and max_attempts exhausted without reaching toBe. If steps completes successfully and waitForField's value becomes toBe, awsWaitForTimedOut == False. Raises: pypyr.errors.KeyNotInContextError: awsWaitFor missing in context. pypyr.errors.KeyInContextHasNoValueError: awsWaitFor exists but is None. pypyraws.errors.WaitTimeOut: maxAttempts exceeded without waitForField changing to toBe. """ logger.debug("started") context.assert_key_has_value('awsWaitFor', __name__) wait_for = context['awsWaitFor'] assert_key_has_value(wait_for, 'awsClientIn', __name__, 'awsWaitFor') aws_client_in = context.get_formatted_value(wait_for['awsClientIn']) (service_name, method_name, client_args, method_args) = contextargs.get_awsclient_args(aws_client_in, __name__) (wait_for_field, to_be, poll_interval, max_attempts, error_on_wait_timeout) = get_poll_args(wait_for, context) wait_response = wait_until_true( interval=poll_interval, max_attempts=max_attempts)(execute_aws_client_method)( service_name=service_name, method_name=method_name, client_args=client_args, method_args=method_args, wait_for_field=wait_for_field, to_be=to_be) if wait_response: context['awsWaitForTimedOut'] = False logger.info(f"aws {service_name} {method_name} returned {to_be}. " "Pipeline will now continue.") else: if error_on_wait_timeout: context['awsWaitForTimedOut'] = True logger.error(f"aws {service_name} {method_name} did not return " f"{to_be} within {max_attempts}. errorOnWaitTimeout " "is True, throwing error") raise WaitTimeOut(f"aws {service_name} {method_name} did not " f"return {to_be} within {max_attempts} retries.") else: context['awsWaitForTimedOut'] = True logger.warning( f"aws {service_name} {method_name} did NOT return " f" {to_be}. errorOnWaitTimeout is False, so pipeline " "will proceed to the next step anyway.") logger.debug("done")
def test_assert_key_value_empty(): """Not a truthy.""" asserts.assert_key_has_value(obj={'k1': ''}, key='k1', caller='arb caller', parent='parent name')
def test_assert_key_has_value_passes(): """Success case.""" asserts.assert_key_has_value(obj={'k1': 'v1'}, key='k1', caller='arb caller', parent='parent name')
def run_step(context): """Load a json file into the pypyr context. json parsed from the file will be merged into the pypyr context. This will overwrite existing values if the same keys are already in there. I.e if file json has {'eggs' : 'boiled'} and context {'eggs': 'fried'} already exists, returned context['eggs'] will be 'boiled'. The json should not be an array [] on the top level, but rather an Object. Args: context: pypyr.context.Context. Mandatory. The following context key must exist - fetchJson - path. path-like. Path to file on disk. - key. string. If exists, write json structure to this context key. Else json writes to context root. Also supports a passing path as string to fetchJson, but in this case you won't be able to specify a key. All inputs support formatting expressions. Returns: None. updates context arg. Raises: FileNotFoundError: take a guess pypyr.errors.KeyNotInContextError: fetchJson.path missing in context. pypyr.errors.KeyInContextHasNoValueError: fetchJson.path exists but is None. """ logger.debug("started") context.assert_key_has_value(key='fetchJson', caller=__name__) fetch_json_input = context.get_formatted('fetchJson') if isinstance(fetch_json_input, str): file_path = fetch_json_input destination_key = None else: assert_key_has_value(obj=fetch_json_input, key='path', caller=__name__, parent='fetchJson') file_path = fetch_json_input['path'] destination_key = fetch_json_input.get('key', None) logger.debug("attempting to open file: %s", file_path) with open(file_path) as json_file: payload = json.load(json_file) if destination_key: logger.debug("json file loaded. Writing to context %s", destination_key) context[destination_key] = payload else: if not isinstance(payload, MutableMapping): raise TypeError( 'json input should describe an object at the top ' 'level when fetchJsonKey isn\'t specified. You should have ' 'something like {"key1": "value1", "key2": "value2"} ' 'in the json top-level, not ["value1", "value2"]') logger.debug("json file loaded. Merging into pypyr context. . .") context.update(payload) logger.info("json file written into pypyr context. Count: %s", len(payload)) logger.debug("done")