def test_decrypt_parent_source_dict(self, sops_mock):
encrypted_config_file_path = os.path.join(
os.path.dirname(__file__), 'files', 'tssc-config-secret-stuff.yml')
encrypted_config = parse_yaml_or_json_file(encrypted_config_file_path)
encrypted_config_json = json.dumps(encrypted_config)
config_value = ConfigValue(
value=
'ENC[AES256_GCM,data:UGKfnzsSrciR7GXZJhOCMmFrz3Y6V3pZsd3P,iv:yuReqA+n+rRXVHMc+2US5t7yPx54sooZSXWV4KLjDIs=,tag:jueP7/ZWLfYrEuhh+4eS8g==,type:str]',
parent_source=encrypted_config,
path_parts=[
'tssc-config', 'global-environment-defaults', 'DEV',
'kube-api-token'
])
sops_decryptor = SOPS()
sops_decryptor.decrypt(config_value)
sops_mock.assert_called_once_with(
'--decrypt',
'--extract=["tssc-config"]["global-environment-defaults"]["DEV"]["kube-api-token"]',
'--input-type=json',
'/dev/stdin',
_in=encrypted_config_json,
_out=Any(StringIO),
_err=Any(StringIO))
def test_decrypt_additional_sops_args(self, sops_mock):
encrypted_config_file_path = os.path.join(
os.path.dirname(__file__), 'files', 'tssc-config-secret-stuff.yml')
config_value = ConfigValue(
value=
'ENC[AES256_GCM,data:UGKfnzsSrciR7GXZJhOCMmFrz3Y6V3pZsd3P,iv:yuReqA+n+rRXVHMc+2US5t7yPx54sooZSXWV4KLjDIs=,tag:jueP7/ZWLfYrEuhh+4eS8g==,type:str]',
parent_source=encrypted_config_file_path,
path_parts=[
'tssc-config', 'global-environment-defaults', 'DEV',
'kube-api-token'
])
sops_decryptor = SOPS(additional_sops_args=['--aws-profile=foo'])
sops_decryptor.decrypt(config_value)
sops_mock.assert_called_once_with(
'--decrypt',
'--extract=["tssc-config"]["global-environment-defaults"]["DEV"]["kube-api-token"]',
None,
encrypted_config_file_path,
'--aws-profile=foo',
_in=None,
_out=Any(StringIO),
_err=Any(StringIO))
def test_register_obfuscation_stream(self):
secret_value = "decrypt me"
config_value = ConfigValue(f'TEST_ENC[{secret_value}]')
DecryptionUtils.register_config_value_decryptor(
SampleConfigValueDecryptor())
out = io.StringIO()
with redirect_stdout(out):
old_stdout = sys.stdout
new_stdout = TextIOSelectiveObfuscator(old_stdout)
DecryptionUtils.register_obfuscation_stream(new_stdout)
try:
sys.stdout = new_stdout
DecryptionUtils.decrypt(config_value)
print(
f"ensure that I can't actually leak secret value ({secret_value})"
)
self.assertRegex(
out.getvalue(),
r"ensure that I can't actually leak secret value \(\*+\)")
finally:
new_stdout.close()
sys.stdout = old_stdout
def test_decrypt_sample_decryptor_does_not_match(self):
config_value = ConfigValue('attempt to decrypt me')
DecryptionUtils.register_config_value_decryptor(
SampleConfigValueDecryptor())
decrypted_value = DecryptionUtils.decrypt(config_value)
self.assertIsNone(decrypted_value)
def test_create_and_register_config_value_decryptor_no_constructor_args(
self):
DecryptionUtils.create_and_register_config_value_decryptor(
'tests.test_decryption_utils.SampleConfigValueDecryptor')
secret_value = "decrypt me"
config_value = ConfigValue(f'TEST_ENC[{secret_value}]')
decrypted_value = DecryptionUtils.decrypt(config_value)
self.assertEqual(decrypted_value, secret_value)
def test_decrypt_sample_decryptor(self):
secret_value = "decrypt me"
config_value = ConfigValue(f'TEST_ENC[{secret_value}]')
DecryptionUtils.register_config_value_decryptor(
SampleConfigValueDecryptor())
decrypted_value = DecryptionUtils.decrypt(config_value)
self.assertEqual(decrypted_value, secret_value)
def test_get_sops_value_path(self, sops_mock):
config_value = ConfigValue(
value=
'ENC[AES256_GCM,data:UGKfnzsSrciR7GXZJhOCMmFrz3Y6V3pZsd3P,iv:yuReqA+n+rRXVHMc+2US5t7yPx54sooZSXWV4KLjDIs=,tag:jueP7/ZWLfYrEuhh+4eS8g==,type:str]',
parent_source=None,
path_parts=["tssc-config", "step-foo", 0, "config", "test1"])
sops_value_path = SOPS.get_sops_value_path(config_value)
self.assertEqual(sops_value_path,
'["tssc-config"]["step-foo"][0]["config"]["test1"]')
def test_create_and_register_config_value_decryptor_required_constructor_args(
self):
DecryptionUtils.create_and_register_config_value_decryptor(
'tests.test_decryption_utils.RequiredConstructorParamsConfigValueDecryptor',
{'required_arg': 'hello world'})
secret_value = "decrypt me"
config_value = ConfigValue(f'TEST_ENC[{secret_value}]')
decrypted_value = DecryptionUtils.decrypt(config_value)
self.assertEqual(decrypted_value, secret_value)
def test_can_can_decrypt_not_string(self, sops_mock):
encrypted_config_file_path = os.path.join(
os.path.dirname(__file__), 'files', 'tssc-config-secret-stuff.yml')
config_value = ConfigValue(value=True,
parent_source=encrypted_config_file_path,
path_parts=[
'tssc-config',
'global-environment-defaults', 'DEV',
'kube-api-token'
])
sops_decryptor = SOPS()
self.assertFalse(sops_decryptor.can_decrypt(config_value))
def test_can_decrypt_true(self):
encrypted_config_file_path = os.path.join(
os.path.dirname(__file__), 'files', 'tssc-config-secret-stuff.yml')
config_value = ConfigValue(
value=
'ENC[AES256_GCM,data:UGKfnzsSrciR7GXZJhOCMmFrz3Y6V3pZsd3P,iv:yuReqA+n+rRXVHMc+2US5t7yPx54sooZSXWV4KLjDIs=,tag:jueP7/ZWLfYrEuhh+4eS8g==,type:str]',
parent_source=encrypted_config_file_path,
path_parts=[
'tssc-config', 'global-environment-defaults', 'DEV',
'kube-api-token'
])
sops_decryptor = SOPS()
self.assertTrue(sops_decryptor.can_decrypt(config_value))
def test_decrypt_parent_source_none(self):
config_value = ConfigValue(
value=
'ENC[AES256_GCM,data:UGKfnzsSrciR7GXZJhOCMmFrz3Y6V3pZsd3P,iv:yuReqA+n+rRXVHMc+2US5t7yPx54sooZSXWV4KLjDIs=,tag:jueP7/ZWLfYrEuhh+4eS8g==,type:str]',
parent_source=None,
path_parts=[
'tssc-config', 'global-environment-defaults', 'DEV',
'kube-api-token'
])
sops_decryptor = SOPS()
with self.assertRaisesRegex(
ValueError,
r"Given config value \(ConfigValue\(.*\)\) parent source \(None\) " \
r"is expected to be of type dict or str but is of type: <class 'NoneType'>"
):
sops_decryptor.decrypt(config_value)
def test_decrypt_parent_source_file_does_not_exist(self):
config_value = ConfigValue(
value=
'ENC[AES256_GCM,data:UGKfnzsSrciR7GXZJhOCMmFrz3Y6V3pZsd3P,iv:yuReqA+n+rRXVHMc+2US5t7yPx54sooZSXWV4KLjDIs=,tag:jueP7/ZWLfYrEuhh+4eS8g==,type:str]',
parent_source='does-not-exist.yml',
path_parts=[
'tssc-config', 'global-environment-defaults', 'DEV',
'kube-api-token'
])
sops_decryptor = SOPS()
with self.assertRaisesRegex(
ValueError,
r"Given config value \(ConfigValue\(.*\)\) parent source \(does-not-exist.yml\)" \
r" is of type \(str\) but is not a path to a file that exists"
):
sops_decryptor.decrypt(config_value)
def test_decrypt_sops_error(self, sops_mock):
encrypted_config_file_path = os.path.join(
os.path.dirname(__file__), 'files', 'tssc-config-secret-stuff.yml')
config_value = ConfigValue(
value=
'ENC[AES256_GCM,data:UGKfnzsSrciR7GXZJhOCMmFrz3Y6V3pZsd3P,iv:yuReqA+n+rRXVHMc+2US5t7yPx54sooZSXWV4KLjDIs=,tag:jueP7/ZWLfYrEuhh+4eS8g==,type:str]',
parent_source=encrypted_config_file_path,
path_parts=[
'tssc-config', 'global-environment-defaults', 'DEV',
'kube-api-token'
])
sops_decryptor = SOPS()
sh.sops.side_effect = sh.ErrorReturnCode(
'sops', b'mock stdout', b'mock error about issue running sops')
with self.assertRaisesRegex(
RuntimeError,
r"Error invoking sops when trying to decrypt config value \(ConfigValue\(.*\)\):"
):
sops_decryptor.decrypt(config_value)
def test_decrypt_no_valid_key(self):
encrypted_config_file_path = os.path.join(
os.path.dirname(__file__), 'files', 'tssc-config-secret-stuff.yml')
config_value = ConfigValue(
value=
'ENC[AES256_GCM,data:UGKfnzsSrciR7GXZJhOCMmFrz3Y6V3pZsd3P,iv:yuReqA+n+rRXVHMc+2US5t7yPx54sooZSXWV4KLjDIs=,tag:jueP7/ZWLfYrEuhh+4eS8g==,type:str]',
parent_source=encrypted_config_file_path,
path_parts=[
'tssc-config', 'global-environment-defaults', 'DEV',
'kube-api-token'
])
sops_decryptor = SOPS()
# delete the gpg key needed to decrypt the value
self.delete_gpg_key()
# attempt to decrypt the value
with self.assertRaisesRegex(
RuntimeError,
r"Error invoking sops when trying to decrypt config value \(ConfigValue\(.*\)\):"
):
sops_decryptor.decrypt(config_value)
def __add_config_dict(self, config_dict, source_file_path=None): # pylint: disable=too-many-locals, too-many-branches
"""Add a TSSC configuration dictionary to the list of TSSC configuration dictionaries.
Parameters
----------
config_dict : dict
A dictionary to validate as a TSSC configuration and to add to this Config.
source_file_path : str, optional
File path to the file from which the given config_dict came from.
Raises
------
AssertionError
If the given config_dict is not a valid TSSC configuration dictionary.
If attempt to update an existing sub step and new and existing sub step implementers
do not match.
If sub step does not define a step implementer.
ValueError
If duplicative leaf keys when merging global defaults
If duplicative leaf keys when merging global env defaults
If step config is not of type dict or list
If new sub step configuration has duplicative leaf keys to
existing sub step configuration.
If new sub step environment configuration has duplicative leaf keys to
existing sub step environment configuration.
"""
assert Config.TSSC_CONFIG_KEY in config_dict, \
"Failed to add invalid TSSC config. " + \
f"Missing expected top level key ({Config.TSSC_CONFIG_KEY}): " + \
f"{config_dict}"
# if file path given use that as the source when creating ConfigValue objects
# else use a copy of the given configuration dictionary
if source_file_path is not None:
parent_source = source_file_path
else:
parent_source = copy.deepcopy(config_dict)
# convert all the leaves of the configuration dictionary under
# the Config.TSSC_CONFIG_KEY to ConfigValue objects
tssc_config_values = ConfigValue.convert_leaves_to_config_values(
values=copy.deepcopy(config_dict[Config.TSSC_CONFIG_KEY]),
parent_source=parent_source,
path_parts=[Config.TSSC_CONFIG_KEY]
)
for key, value in tssc_config_values.items():
# if global default key
# else if global env defaults key
# else assume step config
if key == Config.TSSC_CONFIG_KEY_GLOBAL_DEFAULTS:
try:
self.__global_defaults = deep_merge(
copy.deepcopy(self.__global_defaults),
copy.deepcopy(value)
)
except ValueError as error:
raise ValueError(
f"Error merging global defaults: {error}"
) from error
elif key == Config.TSSC_CONFIG_KEY_GLOBAL_ENVIRONMENT_DEFAULTS:
for env, env_config in value.items():
if env not in self.__global_environment_defaults:
self.__global_environment_defaults[env] = {
Config.TSSC_CONFIG_KEY_ENVIRONMENT_NAME: env
}
try:
self.__global_environment_defaults[env] = deep_merge(
copy.deepcopy(self.__global_environment_defaults[env]),
copy.deepcopy(env_config)
)
except ValueError as error:
raise ValueError(
f"Error merging global environment ({env}) defaults: {error}"
) from error
elif key == Config.TSSC_CONFIG_KEY_DECRYPTORS:
config_decryptor_definitions = ConfigValue.convert_leaves_to_values(value)
Config.parse_and_register_decryptors_definitions(config_decryptor_definitions)
else:
step_name = key
step_config = value
# if step_config is dict then assume step with single sub step
if isinstance(step_config, dict):
sub_steps = [step_config]
elif isinstance(step_config, list):
sub_steps = step_config
else:
raise ValueError(
f"Expected step ({step_name}) to have have step config ({step_config})" +
f" of type dict or list but got: {type(step_config)}"
)
for sub_step in sub_steps:
assert Config.TSSC_CONFIG_KEY_STEP_IMPLEMENTER in sub_step, \
f"Step ({step_name}) defines a single sub step with values " + \
f"({sub_step}) but is missing value for key: " + \
f"{Config.TSSC_CONFIG_KEY_STEP_IMPLEMENTER}"
sub_step_implementer_name = \
sub_step[Config.TSSC_CONFIG_KEY_STEP_IMPLEMENTER].value
# if sub step name given
# else if no sub step name given use step implementer as sub step name
if Config.TSSC_CONFIG_KEY_SUB_STEP_NAME in sub_step:
sub_step_name = sub_step[Config.TSSC_CONFIG_KEY_SUB_STEP_NAME].value
else:
sub_step_name = sub_step_implementer_name
if Config.TSSC_CONFIG_KEY_SUB_STEP_CONFIG in sub_step:
sub_step_config = copy.deepcopy(
sub_step[Config.TSSC_CONFIG_KEY_SUB_STEP_CONFIG])
else:
sub_step_config = {}
if Config.TSSC_CONFIG_KEY_SUB_STEP_ENVIRONMENT_CONFIG in sub_step:
sub_step_env_config = copy.deepcopy(
sub_step[Config.TSSC_CONFIG_KEY_SUB_STEP_ENVIRONMENT_CONFIG])
else:
sub_step_env_config = {}
self.add_or_update_step_config(
step_name=step_name,
sub_step_name=sub_step_name,
sub_step_implementer_name=sub_step_implementer_name,
sub_step_config=sub_step_config,
sub_step_env_config=sub_step_env_config
)
def run_step(self):
"""
Wrapper for running the implemented step.
"""
StepImplementer.__print_section_title(f"Step Start - {self.step_name}")
# print information about theconfiguration
StepImplementer.__print_section_title(
f"Configuration - {self.step_name}",
div_char="-",
indent=1
)
StepImplementer.__print_data(
"Step Implementer Configuration Defaults",
ConfigValue.convert_leaves_to_values(self.step_implementer_config_defaults())
)
StepImplementer.__print_data(
"Global Configuration Defaults",
ConfigValue.convert_leaves_to_values(self.global_config_defaults)
)
StepImplementer.__print_data(
"Global Environment Configuration Defaults",
ConfigValue.convert_leaves_to_values(self.global_environment_config_defaults)
)
StepImplementer.__print_data(
"Step Configuration",
ConfigValue.convert_leaves_to_values(self.step_config)
)
StepImplementer.__print_data(
"Step Environment Configuration",
ConfigValue.convert_leaves_to_values(self.step_environment_config)
)
StepImplementer.__print_data(
"Step Configuration Runtime Overrides",
ConfigValue.convert_leaves_to_values(self.step_config_overrides)
)
# create the munged runtime step configuration and print
copy_of_runtime_step_config = self.get_copy_of_runtime_step_config()
StepImplementer.__print_data(
"Runtime Step Configuration",
ConfigValue.convert_leaves_to_values(copy_of_runtime_step_config)
)
# validate the runtime step configuration
StepImplementer.__print_section_title(
f"Standard Out - {self.step_name}",
div_char="-",
indent=1
)
self._validate_runtime_step_config(copy_of_runtime_step_config)
# run the step and save the results
indented_stdout = TextIOIndenter(
parent_stream=sys.stdout,
indent_level=2
)
indented_stderr = TextIOIndenter(
parent_stream=sys.stderr,
indent_level=2
)
with redirect_stdout(indented_stdout), redirect_stderr(indented_stderr):
results = self._run_step()
self.write_results(results)
# print the step run results
StepImplementer.__print_section_title(
f"Results - {self.step_name}",
div_char="-",
indent=1
)
StepImplementer.__print_data('Results File Path', self.results_file_path)
StepImplementer.__print_data('Results', results)
StepImplementer.__print_section_title(f"Step End - {self.step_name}")
def _run_step(self):
runtime_step_config = self.config.get_copy_of_runtime_step_config(
self.environment, self.step_implementer_config_defaults())
return ConfigValue.convert_leaves_to_values(runtime_step_config)
def test_decrypt_no_decryptors(self):
config_value = ConfigValue('attempt to decrypt me')
decrypted_value = DecryptionUtils.decrypt(config_value)
self.assertIsNone(decrypted_value)