def test_jsonpath_replace_with_different_patterns_recursive(self):
"""Edge case to validate that different regexes that live recursively
under the same parent path are handled gracefully. Note that
non-matching regexes are obviously skipped over.
"""
path = ".values"
# Only the first string's pattern will be replaced since it'll match
# REGEX. The second one won't as its pattern is XEGER.
body = {"values": [{"re1": "REGEX_ONE", "nested": ["XEGER_TWO"]}]}
expected = {"values": [{"re1": "YES_ONE", "nested": ["XEGER_TWO"]}]}
result1 = utils.jsonpath_replace(body,
"YES",
jsonpath=path,
pattern="REGEX",
recurse={'depth': -1})
self.assertEqual(expected, result1)
# Now replace the second one by passing in pattern="XEGER".
expected = {"values": [{"re1": "YES_ONE", "nested": ["NO_TWO"]}]}
result2 = utils.jsonpath_replace(result1,
"NO",
jsonpath=path,
pattern="XEGER",
recurse={'depth': -1})
self.assertEqual(expected, result2)
def test_jsonpath_replace_with_recursion_depth_specified(self):
# Only the first string's pattern will be replaced since it'll
# only recurse 1 level.
body = {"re1": "REGEX_ONE", "values": {"re2": "REGEX_TWO"}}
expected = {"re1": "YES_ONE", "values": {"re2": "REGEX_TWO"}}
result = utils.jsonpath_replace(body,
"YES",
jsonpath="$",
pattern="REGEX",
recurse={'depth': 1})
self.assertEqual(expected, result)
# Depth of 2 should cover both.
body = {"re1": "REGEX_ONE", "values": {"re2": "REGEX_TWO"}}
expected = {"re1": "YES_ONE", "values": {"re2": "YES_TWO"}}
result = utils.jsonpath_replace(body,
"YES",
jsonpath="$",
pattern="REGEX",
recurse={'depth': 2})
self.assertEqual(expected, result)
# Depth of 3 is required as the list around "REGEX_TWO" results in
# another layer of recursion.
body = {"re1": "REGEX_ONE", "values": {"re2": ["REGEX_TWO"]}}
expected = {"re1": "YES_ONE", "values": {"re2": ["YES_TWO"]}}
result = utils.jsonpath_replace(body,
"YES",
jsonpath="$",
pattern="REGEX",
recurse={'depth': 3})
self.assertEqual(expected, result)
def test_jsonpath_replace_with_array_index_creates_array(self):
path = ".values.endpoints[0].admin"
expected = {'values': {'endpoints': [{'admin': 'foo'}]}}
result = utils.jsonpath_replace({}, 'foo', path)
self.assertEqual(expected, result)
path = ".values.endpoints[1].admin"
expected = {'values': {'endpoints': [{}, {'admin': 'foo'}]}}
result = utils.jsonpath_replace({}, 'foo', path)
self.assertEqual(expected, result)
def sanitize_potential_secrets(error, document):
"""Sanitize all secret data that may have been substituted into the
document or contained in the document itself (if the document has
``metadata.storagePolicy`` == 'encrypted'). Uses references in
``document.substitutions`` to determine which values to sanitize. Only
meaningful to call this on post-rendered documents.
:param error: Error message produced by ``jsonschema``.
:param document: Document to sanitize.
:type document: DocumentDict
"""
if not document.substitutions and not document.is_encrypted:
return document
to_sanitize = copy.deepcopy(document)
safe_message = 'Sanitized to avoid exposing secret.'
# Sanitize any secrets contained in `error.message` referentially.
if error.message and any(
r.match(error.message)
for r in SecretsSubstitution._insecure_reg_exps):
error.message = safe_message
# Sanitize any secrets extracted from the document itself.
for sub in document.substitutions:
replaced_data = utils.jsonpath_replace(to_sanitize['data'],
safe_message,
sub['dest']['path'])
if replaced_data:
to_sanitize['data'] = replaced_data
return to_sanitize
def test_jsonpath_parse_replace_cache(self):
"""Validate caching for both parsing and replacing functions."""
path = ".values.endpoints.admin"
expected = {'values': {'endpoints': {'admin': 'foo'}}}
# Mock jsonpath_ng to return a monkey-patched parse function that
# keeps track of call count and yet calls the actual function.
with mock.patch.object(
utils,
'jsonpath_ng', # noqa: H210
parse=self.fake_jsonpath_ng):
# Though this is called 3 times, the cached function should only
# be called once, with the cache returning the cached value early.
for _ in range(3):
result = utils.jsonpath_replace({}, 'foo', path)
self.assertEqual(expected, result)
# Though this is called 3 times, the cached function should only
# be called once, with the cache returning the cached value early.
for _ in range(3):
result = utils.jsonpath_parse(expected, path)
self.assertEqual('foo', result)
# Assert that the actual function was called <= 1 times. (Allow for 0
# in case CI jobs clash.)
self.assertThat(self.jsonpath_call_count,
MatchesAny(Equals(0), Equals(1)))
def test_jsonpath_replace_with_pattern(self):
path = ".values.endpoints.admin"
body = {"values": {"endpoints": {"admin": "REGEX_FRESH"}}}
expected = {"values": {"endpoints": {"admin": "EAT_FRESH"}}}
result = utils.jsonpath_replace(body,
"EAT",
jsonpath=path,
pattern="REGEX")
self.assertEqual(expected, result)
def test_jsonpath_replace_with_pattern_and_array_index(self):
path = ".values.endpoints.admin[1]"
body = {"values": {"endpoints": {"admin": [None, "REGEX_FRESH"]}}}
expected = {"values": {"endpoints": {"admin": [None, "EAT_FRESH"]}}}
result = utils.jsonpath_replace(body,
"EAT",
jsonpath=path,
pattern="REGEX")
self.assertEqual(expected, result)
def test_jsonpath_replace_with_pattern_recursive_dict(self):
path = ".values"
body = {"values": {"re1": "REGEX_ONE", "re2": "REGEX_TWO"}}
expected = {"values": {"re1": "YES_ONE", "re2": "YES_TWO"}}
result = utils.jsonpath_replace(body,
"YES",
jsonpath=path,
pattern="REGEX",
recurse={'depth': -1})
self.assertEqual(expected, result)
def test_abstract_document_not_validated(self, mock_log):
test_document = self._read_data('sample_passphrase')
# Set the document to abstract.
abstract_document = utils.jsonpath_replace(
test_document, True, '.metadata.layeringDefinition.abstract')
document_validation.DocumentValidation(
abstract_document, pre_validate=False).validate_all()
self.assertTrue(mock_log.info.called)
self.assertIn("Skipping schema validation for abstract document",
mock_log.info.mock_calls[0][1][0])
def test_jsonpath_replace_with_src_pattern_implicit_match_group_0(self):
src = 'repo.example.com/image:v1.2.3'
src_pattern = '^(.*):(.*)'
path = ".values.image"
body = {"values": {}}
expected = {"values": {"image": "repo.example.com/image:v1.2.3"}}
result = utils.jsonpath_replace(body,
src,
jsonpath=path,
src_pattern=src_pattern)
self.assertEqual(expected, result)
def test_jsonpath_replace_with_pattern_recursive_root_path(self):
"""Validate that recursion happens even from root path."""
path = "$"
body = {"values": {"re1": "REGEX_ONE", "nested": {"re2": "REGEX_TWO"}}}
expected = {"values": {"re1": "YES_ONE", "nested": {"re2": "YES_TWO"}}}
result = utils.jsonpath_replace(body,
"YES",
jsonpath=path,
pattern="REGEX",
recurse={'depth': -1})
self.assertEqual(expected, result)
def test_jsonpath_replace_with_src_pattern(self):
src = 'repo.example.com/image@sha256:e3b0c44298fc...'
src_pattern = 'sha256.*'
path = ".values.image.sha"
body = {"values": {}}
expected = {"values": {"image": {"sha": "sha256:e3b0c44298fc..."}}}
result = utils.jsonpath_replace(body,
src,
jsonpath=path,
src_pattern=src_pattern)
self.assertEqual(expected, result)
def test_jsonpath_replace_with_pattern_recursive_list(self):
path = ".values"
# String entries inside list.
body = {"values": ["REGEX_ONE", "REGEX_TWO"]}
expected = {"values": ["YES_ONE", "YES_TWO"]}
result = utils.jsonpath_replace(body,
"YES",
jsonpath=path,
pattern="REGEX",
recurse={'depth': -1})
self.assertEqual(expected, result)
# Dictionary entries inside list.
body = {"values": [{"re1": "REGEX_ONE", "re2": "REGEX_TWO"}]}
expected = {"values": [{"re1": "YES_ONE", "re2": "YES_TWO"}]}
result = utils.jsonpath_replace(body,
"YES",
jsonpath=path,
pattern="REGEX",
recurse={'depth': -1})
self.assertEqual(expected, result)
def test_jsonpath_replace_with_pattern_recursive_str(self):
"""Edge case to validate that passing in a path that leads to a string
value itself (not a list or dict) still results in pattern replacement
gracefully passing, even though no recursion is technically possible.
"""
path = ".values.endpoints.admin"
body = {"values": {"endpoints": {"admin": "REGEX_FRESH"}}}
expected = {"values": {"endpoints": {"admin": "EAT_FRESH"}}}
result = utils.jsonpath_replace(body,
"EAT",
jsonpath=path,
pattern="REGEX",
recurse={'depth': -1})
self.assertEqual(expected, result)
def test_jsonpath_replace_with_src_pattern_match_group_1(self):
src = 'repo.example.com/image:v1.2.3'
src_pattern = '^(.*):(.*)'
src_match_group = 1
path = ".values.image.repository"
body = {"values": {}}
expected = {
"values": {
"image": {
"repository": "repo.example.com/image"
}
}
}
result = utils.jsonpath_replace(body,
src,
jsonpath=path,
src_pattern=src_pattern,
src_match_group=src_match_group)
self.assertEqual(expected, result)
def _substitute_one(self,
document,
src_doc,
src_secret,
dest_path,
dest_pattern,
dest_recurse=None):
dest_recurse = dest_recurse or {}
exc_message = ''
try:
substituted_data = utils.jsonpath_replace(document.data,
src_secret,
dest_path,
pattern=dest_pattern,
recurse=dest_recurse)
if (isinstance(document.data, dict)
and isinstance(substituted_data, dict)):
document.data.update(substituted_data)
elif substituted_data:
document.data = substituted_data
else:
exc_message = ('Failed to create JSON path "%s" in the '
'destination document [%s, %s] %s. '
'No data was substituted.' %
(dest_path, document.schema, document.layer,
document.name))
except Exception as e:
LOG.error(
'Unexpected exception occurred '
'while attempting '
'substitution using '
'source document [%s, %s] %s '
'referenced in [%s, %s] %s. Details: %s', src_doc.schema,
src_doc.name, src_doc.layer, document.schema, document.layer,
document.name, six.text_type(e))
exc_message = six.text_type(e)
finally:
if exc_message:
self._handle_unknown_substitution_exc(exc_message, src_doc,
document)
return document
def substitutions(self, value):
return utils.jsonpath_replace(self, value, 'metadata.substitutions')
def _apply_action(self, action, child_data, overall_data):
"""Apply actions to each layer that is rendered.
Supported actions include:
* ``merge`` - a "deep" merge that layers new and modified data onto
existing data
* ``replace`` - overwrite data at the specified path and replace it
with the data given in this document
* ``delete`` - remove the data at the specified path
:raises UnsupportedActionMethod: If the layering action isn't found
among ``self.SUPPORTED_METHODS``.
:raises MissingDocumentKey: If a layering action path isn't found
in the child document.
"""
method = action['method']
if method not in self._SUPPORTED_METHODS:
raise errors.UnsupportedActionMethod(
action=action, document=child_data)
# Use copy to prevent these data from being updated referentially.
overall_data = copy.deepcopy(overall_data)
child_data = copy.deepcopy(child_data)
# If None is used, then consider it as a placeholder and coerce the
# data into a dictionary.
if overall_data is None:
overall_data = {}
if child_data is None:
child_data = {}
action_path = action['path']
if action_path.startswith('.data'):
action_path = action_path[5:]
elif action_path.startswith('$.data'):
action_path = action_path[6:]
if not (action_path.startswith('.') or action_path.startswith('$.')):
action_path = '.' + action_path
if method == self._DELETE_ACTION:
if action_path == '.':
overall_data.data = {}
else:
from_child = utils.jsonpath_parse(overall_data.data,
action_path)
if from_child is None:
raise errors.MissingDocumentKey(
child_schema=child_data.schema,
child_layer=child_data.layer,
child_name=child_data.name,
parent_schema=overall_data.schema,
parent_layer=overall_data.layer,
parent_name=overall_data.name,
action=action)
engine_utils.deep_delete(from_child, overall_data.data, None)
elif method == self._MERGE_ACTION:
from_overall = utils.jsonpath_parse(overall_data.data, action_path)
from_child = utils.jsonpath_parse(child_data.data, action_path)
if from_child is None:
raise errors.MissingDocumentKey(
child_schema=child_data.schema,
child_layer=child_data.layer,
child_name=child_data.name,
parent_schema=overall_data.schema,
parent_layer=overall_data.layer,
parent_name=overall_data.name,
action=action)
# If both the child and parent data are dictionaries, then
# traditional merging is possible using JSON path resolution.
# Otherwise, JSON path resolution is not possible, so the only
# way to perform layering is to prioritize the child data over
# that of the parent. This applies when the child data is a
# non-dict, the parent data is a non-dict, or both.
if all(isinstance(x, dict) for x in (from_overall, from_child)):
engine_utils.deep_merge(from_overall, from_child)
else:
LOG.info('Child data is type: %s for [%s, %s] %s. Parent data '
'is type: %s for [%s, %s] %s. Both must be '
'dictionaries for regular JSON path merging to work. '
'Because this is not the case, child data will be '
'prioritized over parent data for "merge" action.',
type(from_child), child_data.schema, child_data.layer,
child_data.name, type(from_overall),
overall_data.schema, overall_data.layer,
overall_data.name)
from_overall = from_child
if from_overall is not None:
overall_data.data = utils.jsonpath_replace(
overall_data.data, from_overall, action_path)
else:
overall_data.data = utils.jsonpath_replace(
overall_data.data, from_child, action_path)
elif method == self._REPLACE_ACTION:
from_child = utils.jsonpath_parse(child_data.data, action_path)
if from_child is None:
raise errors.MissingDocumentKey(
child_schema=child_data.schema,
child_layer=child_data.layer,
child_name=child_data.name,
parent_schema=overall_data.schema,
parent_layer=overall_data.layer,
parent_name=overall_data.name,
action=action)
overall_data.data = utils.jsonpath_replace(
overall_data.data, from_child, action_path)
return overall_data
def test_jsonpath_replace_with_numeric_value_creates_object(self):
path = ".values.endpoints0.admin"
expected = {'values': {'endpoints0': {'admin': 'foo'}}}
result = utils.jsonpath_replace({}, 'foo', path)
self.assertEqual(expected, result)