def test_convert_leaves_to_config_values_0(self):
source = {
Config.CONFIG_KEY: {
'step-foo': [{
'implementer': 'foo1',
'config': {
'test1': 'foo'
}
}]
}
}
ConfigValue.convert_leaves_to_config_values(
values=source[Config.CONFIG_KEY],
parent_source=source,
path_parts=[Config.CONFIG_KEY])
expected = {
Config.CONFIG_KEY: {
'step-foo': [{
'implementer': ConfigValue('foo1', None, None),
'config': {
'test1': ConfigValue('foo', None, None)
}
}]
}
}
self.assertEqual(source, expected)
def test_global_defaults(self):
config = Config({
Config.CONFIG_KEY: {
'global-defaults': {
'test1': 'global-default-1',
'test2': 'global-default-2'
},
'step-foo': [
{
'implementer': 'foo1',
'config': {
'test1': 'foo'
}
}
]
}
})
step_config = config.get_step_config('step-foo')
sub_step = step_config.get_sub_step('foo1')
self.assertEqual(sub_step.global_defaults, {
'test1': ConfigValue(
'global-default-1',
None,
["step-runner-config", "global-defaults", "test1"]
),
'test2': ConfigValue(
'global-default-2',
None,
["step-runner-config", "global-defaults", "test2"]
)
})
def test_convert_leaves_to_values_mixed_leaves(self):
source_values = {
Config.CONFIG_KEY: {
'step-foo': [{
'implementer': ConfigValue('foo1', None, None),
'config': {
'test1': ConfigValue('foo', None, None),
'test': 'not a config value object'
}
}]
}
}
converted = ConfigValue.convert_leaves_to_values(source_values)
self.assertEqual(
converted, {
Config.CONFIG_KEY: {
'step-foo': [{
'implementer': 'foo1',
'config': {
'test1': 'foo',
'test': 'not a config value object'
}
}]
}
})
def test_global_environment_defaults(self):
config = Config({
Config.CONFIG_KEY: {
'global-environment-defaults' : {
'env1': {
'test1': 'env1',
'test2': 'test2'
},
'env2': {
'test1': 'env2',
'test3': 'test3'
}
}
}
})
self.assertEqual(config.global_defaults, {})
self.assertEqual(
ConfigValue.convert_leaves_to_values(config.global_environment_defaults),
{
'env1': {
'environment-name' : 'env1',
'test1' : 'env1',
'test2' : 'test2'
},
'env2': {
'environment-name' : 'env2',
'test1' : 'env2',
'test3' : 'test3'
}
}
)
self.assertEqual(
ConfigValue.convert_leaves_to_values(
config.get_global_environment_defaults_for_environment('env1')
),
{
'environment-name' : 'env1',
'test1' : 'env1',
'test2' : 'test2'
}
)
self.assertEqual(
ConfigValue.convert_leaves_to_values(
config.get_global_environment_defaults_for_environment('env2')
), {
'environment-name' : 'env2',
'test1' : 'env2',
'test3' : 'test3'
}
)
self.assertEqual(config.get_global_environment_defaults_for_environment('does-not-exist'), {})
self.assertEqual(config.get_global_environment_defaults_for_environment(None), {})
def _generate_maven_settings(self):
maven_servers = ConfigValue.convert_leaves_to_values(
self.get_value('maven-servers'))
maven_repositories = ConfigValue.convert_leaves_to_values(
self.get_value('maven-repositories'))
maven_mirrors = ConfigValue.convert_leaves_to_values(
self.get_value('maven-mirrors'))
return generate_maven_settings(working_dir=self.work_dir_path,
maven_servers=maven_servers,
maven_repositories=maven_repositories,
maven_mirrors=maven_mirrors)
def test_global_environment_defaults(self):
config = Config({
Config.CONFIG_KEY: {
'global-environment-defaults': {
'env1': {
'test1': 'global-env1-default-1',
'test2': 'global-env1-default-2'
},
'env2': {
'test1': 'global-env2-default-1',
'test2': 'global-env2-default-2'
}
},
'step-foo': [
{
'implementer': 'foo1',
'config': {
'test1': 'foo'
}
}
]
}
})
step_config = config.get_step_config('step-foo')
sub_step = step_config.get_sub_step('foo1')
self.assertEqual(
ConfigValue.convert_leaves_to_values(
sub_step.get_global_environment_defaults('env1')
),
{
'environment-name': 'env1',
'test1': 'global-env1-default-1',
'test2': 'global-env1-default-2'
}
)
self.assertEqual(
ConfigValue.convert_leaves_to_values(
sub_step.get_global_environment_defaults('env2')
),
{
'environment-name': 'env2',
'test1': 'global-env2-default-1',
'test2': 'global-env2-default-2'
}
)
def test_register_obfuscation_stream(self):
secret_value = "decrypt me"
config_value = ConfigValue(f'TEST_ENC[{secret_value}]')
DecryptionUtils.register_config_value_decryptor(
SampleConfigValueDecryptor())
out = io.StringIO()
with redirect_stdout(out):
old_stdout = sys.stdout
new_stdout = TextIOSelectiveObfuscator(old_stdout)
DecryptionUtils.register_obfuscation_stream(new_stdout)
try:
sys.stdout = new_stdout
DecryptionUtils.decrypt(config_value)
print(
f"ensure that I can't actually leak secret value ({secret_value})"
)
self.assertRegex(
out.getvalue(),
r"ensure that I can't actually leak secret value \(\*+\)")
finally:
new_stdout.close()
sys.stdout = old_stdout
def test_value_decyrpt(self, sops_mock):
encrypted_config_file_path = os.path.join(
os.path.dirname(__file__), 'decryptors', 'files',
'step-runner-config-secret-stuff.yml')
config_value = ConfigValue(
value=
'ENC[AES256_GCM,data:UGKfnzsSrciR7GXZJhOCMmFrz3Y6V3pZsd3P,iv:yuReqA+n+rRXVHMc+2US5t7yPx54sooZSXWV4KLjDIs=,tag:jueP7/ZWLfYrEuhh+4eS8g==,type:str]',
parent_source=encrypted_config_file_path,
path_parts=[
'step-runner-config', 'global-environment-defaults', 'DEV',
'kube-api-token'
])
DecryptionUtils.register_config_value_decryptor(SOPS())
sops_mock.side_effect = create_sops_side_effect('mock decrypted value')
decrypted_value = config_value.value
sops_mock.assert_called_once_with(
'--decrypt',
'--extract=["step-runner-config"]["global-environment-defaults"]["DEV"]["kube-api-token"]',
None,
encrypted_config_file_path,
_in=None,
_out=Any(StringIO),
_err=Any(StringIO))
self.assertEqual(decrypted_value, 'mock decrypted value')
def test_decrypt_no_valid_key(self):
encrypted_config_file_path = os.path.join(
os.path.dirname(__file__), 'files',
'step-runner-config-secret-stuff.yml')
config_value = ConfigValue(
value=
'ENC[AES256_GCM,data:UGKfnzsSrciR7GXZJhOCMmFrz3Y6V3pZsd3P,iv:yuReqA+n+rRXVHMc+2US5t7yPx54sooZSXWV4KLjDIs=,tag:jueP7/ZWLfYrEuhh+4eS8g==,type:str]',
parent_source=encrypted_config_file_path,
path_parts=[
'step-runner-config', 'global-environment-defaults', 'DEV',
'kube-api-token'
])
sops_decryptor = SOPS()
# delete the gpg key needed to decrypt the value
self.delete_gpg_key()
# attempt to decrypt the value
with self.assertRaisesRegex(
RuntimeError,
r"Error invoking sops when trying to decrypt config value \(ConfigValue\(.*\)\):"
):
sops_decryptor.decrypt(config_value)
def test_get_sub_step_env_config(self):
config = Config({
Config.CONFIG_KEY: {
'step-foo': [
{
'implementer': 'foo1',
'config': {
'test1': 'foo'
},
'environment-config': {
'env1': {
'test2': 'bar'
}
}
}
]
}
})
step_config = config.get_step_config('step-foo')
sub_step = step_config.get_sub_step('foo1')
self.assertEqual(
ConfigValue.convert_leaves_to_values(
sub_step.get_sub_step_env_config('env1')
),
{
'test2': 'bar'
}
)
def test_decrypt_parent_source_dict(self, sops_mock):
encrypted_config_file_path = os.path.join(
os.path.dirname(__file__), 'files',
'step-runner-config-secret-stuff.yml')
encrypted_config = parse_yaml_or_json_file(encrypted_config_file_path)
encrypted_config_json = json.dumps(encrypted_config)
config_value = ConfigValue(
value=
'ENC[AES256_GCM,data:UGKfnzsSrciR7GXZJhOCMmFrz3Y6V3pZsd3P,iv:yuReqA+n+rRXVHMc+2US5t7yPx54sooZSXWV4KLjDIs=,tag:jueP7/ZWLfYrEuhh+4eS8g==,type:str]',
parent_source=encrypted_config,
path_parts=[
'step-runner-config', 'global-environment-defaults', 'DEV',
'kube-api-token'
])
sops_decryptor = SOPS()
sops_decryptor.decrypt(config_value)
sops_mock.assert_called_once_with(
'--decrypt',
'--extract=["step-runner-config"]["global-environment-defaults"]["DEV"]["kube-api-token"]',
'--input-type=json',
'/dev/stdin',
_in=encrypted_config_json,
_out=Any(StringIO),
_err=Any(StringIO))
def test_sub_step_with_no_environment_config(self):
config = Config({
Config.CONFIG_KEY: {
'step-foo': [
{
'implementer': 'foo1',
'config': {
'test1': 'foo'
}
}
]
}
})
step_config = config.get_step_config('step-foo')
self.assertEqual(len(step_config.sub_steps), 1)
self.assertEqual(
ConfigValue.convert_leaves_to_values(
step_config.get_sub_step('foo1').sub_step_config
),
{
'test1': 'foo'
}
)
self.assertEqual(step_config.get_sub_step('foo1').get_sub_step_env_config('env1'), {})
def test_decrypt_additional_sops_args(self, sops_mock):
encrypted_config_file_path = os.path.join(
os.path.dirname(__file__), 'files',
'step-runner-config-secret-stuff.yml')
config_value = ConfigValue(
value=
'ENC[AES256_GCM,data:UGKfnzsSrciR7GXZJhOCMmFrz3Y6V3pZsd3P,iv:yuReqA+n+rRXVHMc+2US5t7yPx54sooZSXWV4KLjDIs=,tag:jueP7/ZWLfYrEuhh+4eS8g==,type:str]',
parent_source=encrypted_config_file_path,
path_parts=[
'step-runner-config', 'global-environment-defaults', 'DEV',
'kube-api-token'
])
sops_decryptor = SOPS(additional_sops_args=['--aws-profile=foo'])
sops_decryptor.decrypt(config_value)
sops_mock.assert_called_once_with(
'--decrypt',
'--extract=["step-runner-config"]["global-environment-defaults"]["DEV"]["kube-api-token"]',
None,
encrypted_config_file_path,
'--aws-profile=foo',
_in=None,
_out=Any(StringIO),
_err=Any(StringIO))
def test_sub_step_with_name(self):
config = Config({
Config.CONFIG_KEY: {
'step-foo': [
{
'name': 'sub-step-name-test',
'implementer': 'foo1',
'config': {
'test1': 'foo'
}
}
]
}
})
step_config = config.get_step_config('step-foo')
self.assertEqual(len(step_config.sub_steps), 1)
self.assertEqual(
ConfigValue.convert_leaves_to_values(
step_config.get_sub_step('sub-step-name-test').sub_step_config
),
{
'test1': 'foo'
}
)
def test_add_or_update_sub_step_config_exising_sub_step(self):
config = Config([
{
Config.CONFIG_KEY: {
'step-foo': [{
'implementer': 'foo1',
'config': {
'test1': 'foo'
}
}]
}
},
{
Config.CONFIG_KEY: {
'step-foo': [{
'implementer': 'foo1',
'config': {
'test2': 'foo'
}
}]
}
},
])
step_config = config.get_step_config('step-foo')
self.assertEqual(len(step_config.sub_steps), 1)
self.assertEqual(
ConfigValue.convert_leaves_to_values(
step_config.get_sub_step('foo1').sub_step_config), {
'test1': 'foo',
'test2': 'foo'
})
def test_sub_step_with_continue_sub_steps_on_failure_str(self):
config = Config({
Config.CONFIG_KEY: {
'step-foo': [
{
'implementer': 'foo1',
'continue-sub-steps-on-failure': 'true',
'config': {
'test1': 'foo'
}
},
{
'implementer': 'foo2',
'config': {
'test2': 'foo'
}
}
]
}
})
step_config = config.get_step_config('step-foo')
self.assertEqual(len(step_config.sub_steps), 2)
self.assertEqual(
ConfigValue.convert_leaves_to_values(
step_config.get_sub_step('foo1').sub_step_config,
),
{
'test1': 'foo'
}
)
self.assertEqual(
ConfigValue.convert_leaves_to_values(
step_config.get_sub_step('foo2').sub_step_config
),
{
'test2': 'foo'
}
)
self.assertTrue(
step_config.get_sub_step('foo1').sub_step_contine_sub_steps_on_failure
)
self.assertFalse(
step_config.get_sub_step('foo2').sub_step_contine_sub_steps_on_failure
)
def _run_step(self):
step_result = StepResult.from_step_implementer(self)
runtime_step_config = self.config.get_copy_of_runtime_step_config(
self.environment, self.step_implementer_config_defaults())
for n, v in ConfigValue.convert_leaves_to_values(
runtime_step_config).items():
step_result.add_artifact(name=n, value=v)
return step_result
def test_decrypt_sample_decryptor_does_not_match(self):
config_value = ConfigValue('attempt to decrypt me')
DecryptionUtils.register_config_value_decryptor(
SampleConfigValueDecryptor())
decrypted_value = DecryptionUtils.decrypt(config_value)
self.assertIsNone(decrypted_value)
def test_create_and_register_config_value_decryptor_no_constructor_args(
self):
DecryptionUtils.create_and_register_config_value_decryptor(
'tests.test_decryption_utils.SampleConfigValueDecryptor')
secret_value = "decrypt me"
config_value = ConfigValue(f'TEST_ENC[{secret_value}]')
decrypted_value = DecryptionUtils.decrypt(config_value)
self.assertEqual(decrypted_value, secret_value)
def test_decrypt_sample_decryptor(self):
secret_value = "decrypt me"
config_value = ConfigValue(f'TEST_ENC[{secret_value}]')
DecryptionUtils.register_config_value_decryptor(
SampleConfigValueDecryptor())
decrypted_value = DecryptionUtils.decrypt(config_value)
self.assertEqual(decrypted_value, secret_value)
def test_create_and_register_config_value_decryptor_required_constructor_args(
self):
DecryptionUtils.create_and_register_config_value_decryptor(
'tests.test_decryption_utils.RequiredConstructorParamsConfigValueDecryptor',
{'required_arg': 'hello world'})
secret_value = "decrypt me"
config_value = ConfigValue(f'TEST_ENC[{secret_value}]')
decrypted_value = DecryptionUtils.decrypt(config_value)
self.assertEqual(decrypted_value, secret_value)
def test_value_path_given_no_inital_value_path_parts(self):
source = {
Config.CONFIG_KEY: {
'step-foo': [{
'implementer': 'foo1',
'config': {
'test1': 'foo'
}
}]
}
}
ConfigValue.convert_leaves_to_config_values(values=source,
parent_source=source)
self.assertEqual(
source[Config.CONFIG_KEY]['step-foo'][0]['config']
['test1'].path_parts,
['step-runner-config', 'step-foo', 0, 'config', 'test1'])
def maven_settings_file(self):
"""Gets the maven settings file for this step.
"""
if not self.__maven_settings_file:
maven_servers = ConfigValue.convert_leaves_to_values(
self.get_value('maven-servers'))
maven_repositories = ConfigValue.convert_leaves_to_values(
self.get_value('maven-repositories'))
maven_mirrors = ConfigValue.convert_leaves_to_values(
self.get_value('maven-mirrors'))
self.__maven_settings_file = generate_maven_settings(
working_dir=self.work_dir_path,
maven_servers=maven_servers,
maven_repositories=maven_repositories,
maven_mirrors=maven_mirrors)
return self.__maven_settings_file
def _run_step(self):
step_result = StepResult.from_step_implementer(self)
runtime_step_config = self.config.get_copy_of_runtime_step_config(
self.environment, self.step_implementer_config_defaults())
# copy the key/value pairs into the artifacts
for name, value in ConfigValue.convert_leaves_to_values(
runtime_step_config).items():
# print(name, value)
step_result.add_artifact(name, value)
return step_result
def test__repr__(self):
source = {
Config.CONFIG_KEY: {
'step-foo': [{
'implementer': 'foo1',
'config': {
'test1': 'foo'
}
}]
}
}
ConfigValue.convert_leaves_to_config_values(
values=source[Config.CONFIG_KEY],
parent_source=source,
path_parts=[Config.CONFIG_KEY])
self.assertEqual(
str(source[Config.CONFIG_KEY]['step-foo'][0]['config']['test1']),
"ConfigValue(value=foo, value_path='['step-runner-config', 'step-foo', 0, 'config', 'test1']')"
)
def test_multiple_sub_steps(self):
config = Config({
Config.CONFIG_KEY: {
'step-foo': [
{
'implementer': 'foo1',
'config': {
'test1': 'foo'
}
},
{
'implementer': 'foo2',
'config': {
'test2': 'foo'
}
}
]
}
})
step_config = config.get_step_config('step-foo')
self.assertEqual(len(step_config.sub_steps), 2)
self.assertEqual(
ConfigValue.convert_leaves_to_values(
step_config.get_sub_step('foo1').sub_step_config,
),
{
'test1': 'foo'
}
)
self.assertEqual(
ConfigValue.convert_leaves_to_values(
step_config.get_sub_step('foo2').sub_step_config
),
{
'test2': 'foo'
}
)
def test_get_sops_value_path(self, sops_mock):
config_value = ConfigValue(
value=
'ENC[AES256_GCM,data:UGKfnzsSrciR7GXZJhOCMmFrz3Y6V3pZsd3P,iv:yuReqA+n+rRXVHMc+2US5t7yPx54sooZSXWV4KLjDIs=,tag:jueP7/ZWLfYrEuhh+4eS8g==,type:str]',
parent_source=None,
path_parts=[
"step-runner-config", "step-foo", 0, "config", "test1"
])
sops_value_path = SOPS.get_sops_value_path(config_value)
self.assertEqual(
sops_value_path,
'["step-runner-config"]["step-foo"][0]["config"]["test1"]')
def test_merge_valid_global_environment_defaults_same_env_diff_keys(self):
with TempDirectory() as temp_dir:
config_dir = "test"
config_files = [
{
'name': os.path.join(config_dir,'foo.yml'),
'contents' : {
Config.CONFIG_KEY: {
'global-environment-defaults' : {
'env1' : {
'foo-key': 'foo'
}
}
}
}
},
{
'name': os.path.join(config_dir,'bar.yml'),
'contents' : {
Config.CONFIG_KEY: {
'global-environment-defaults' : {
'env1' : {
'bar-key': 'bar'
}
}
}
}
},
]
for config_file in config_files:
config_file_name = config_file['name']
config_file_contents = config_file['contents']
temp_dir.write(
config_file_name,
bytes(f"{config_file_contents}", 'utf-8')
)
config = Config()
config.add_config(os.path.join(temp_dir.path, config_dir))
self.assertEqual(
ConfigValue.convert_leaves_to_values(
config.get_global_environment_defaults_for_environment('env1')
),
{
'environment-name': 'env1',
'foo-key': 'foo',
'bar-key': 'bar'
}
)
def test_can_can_decrypt_not_string(self, sops_mock):
encrypted_config_file_path = os.path.join(
os.path.dirname(__file__), 'files',
'step-runner-config-secret-stuff.yml')
config_value = ConfigValue(value=True,
parent_source=encrypted_config_file_path,
path_parts=[
'step-runner-config',
'global-environment-defaults', 'DEV',
'kube-api-token'
])
sops_decryptor = SOPS()
self.assertFalse(sops_decryptor.can_decrypt(config_value))
def test_can_decrypt_true(self):
encrypted_config_file_path = os.path.join(
os.path.dirname(__file__), 'files',
'step-runner-config-secret-stuff.yml')
config_value = ConfigValue(
value=
'ENC[AES256_GCM,data:UGKfnzsSrciR7GXZJhOCMmFrz3Y6V3pZsd3P,iv:yuReqA+n+rRXVHMc+2US5t7yPx54sooZSXWV4KLjDIs=,tag:jueP7/ZWLfYrEuhh+4eS8g==,type:str]',
parent_source=encrypted_config_file_path,
path_parts=[
'step-runner-config', 'global-environment-defaults', 'DEV',
'kube-api-token'
])
sops_decryptor = SOPS()
self.assertTrue(sops_decryptor.can_decrypt(config_value))
