def test_convert_leaves_to_config_values_0(self): source = { Config.CONFIG_KEY: { 'step-foo': [{ 'implementer': 'foo1', 'config': { 'test1': 'foo' } }] } } ConfigValue.convert_leaves_to_config_values( values=source[Config.CONFIG_KEY], parent_source=source, path_parts=[Config.CONFIG_KEY]) expected = { Config.CONFIG_KEY: { 'step-foo': [{ 'implementer': ConfigValue('foo1', None, None), 'config': { 'test1': ConfigValue('foo', None, None) } }] } } self.assertEqual(source, expected)
def test_global_defaults(self): config = Config({ Config.CONFIG_KEY: { 'global-defaults': { 'test1': 'global-default-1', 'test2': 'global-default-2' }, 'step-foo': [ { 'implementer': 'foo1', 'config': { 'test1': 'foo' } } ] } }) step_config = config.get_step_config('step-foo') sub_step = step_config.get_sub_step('foo1') self.assertEqual(sub_step.global_defaults, { 'test1': ConfigValue( 'global-default-1', None, ["step-runner-config", "global-defaults", "test1"] ), 'test2': ConfigValue( 'global-default-2', None, ["step-runner-config", "global-defaults", "test2"] ) })
def test_convert_leaves_to_values_mixed_leaves(self): source_values = { Config.CONFIG_KEY: { 'step-foo': [{ 'implementer': ConfigValue('foo1', None, None), 'config': { 'test1': ConfigValue('foo', None, None), 'test': 'not a config value object' } }] } } converted = ConfigValue.convert_leaves_to_values(source_values) self.assertEqual( converted, { Config.CONFIG_KEY: { 'step-foo': [{ 'implementer': 'foo1', 'config': { 'test1': 'foo', 'test': 'not a config value object' } }] } })
def test_global_environment_defaults(self): config = Config({ Config.CONFIG_KEY: { 'global-environment-defaults' : { 'env1': { 'test1': 'env1', 'test2': 'test2' }, 'env2': { 'test1': 'env2', 'test3': 'test3' } } } }) self.assertEqual(config.global_defaults, {}) self.assertEqual( ConfigValue.convert_leaves_to_values(config.global_environment_defaults), { 'env1': { 'environment-name' : 'env1', 'test1' : 'env1', 'test2' : 'test2' }, 'env2': { 'environment-name' : 'env2', 'test1' : 'env2', 'test3' : 'test3' } } ) self.assertEqual( ConfigValue.convert_leaves_to_values( config.get_global_environment_defaults_for_environment('env1') ), { 'environment-name' : 'env1', 'test1' : 'env1', 'test2' : 'test2' } ) self.assertEqual( ConfigValue.convert_leaves_to_values( config.get_global_environment_defaults_for_environment('env2') ), { 'environment-name' : 'env2', 'test1' : 'env2', 'test3' : 'test3' } ) self.assertEqual(config.get_global_environment_defaults_for_environment('does-not-exist'), {}) self.assertEqual(config.get_global_environment_defaults_for_environment(None), {})
def _generate_maven_settings(self): maven_servers = ConfigValue.convert_leaves_to_values( self.get_value('maven-servers')) maven_repositories = ConfigValue.convert_leaves_to_values( self.get_value('maven-repositories')) maven_mirrors = ConfigValue.convert_leaves_to_values( self.get_value('maven-mirrors')) return generate_maven_settings(working_dir=self.work_dir_path, maven_servers=maven_servers, maven_repositories=maven_repositories, maven_mirrors=maven_mirrors)
def test_global_environment_defaults(self): config = Config({ Config.CONFIG_KEY: { 'global-environment-defaults': { 'env1': { 'test1': 'global-env1-default-1', 'test2': 'global-env1-default-2' }, 'env2': { 'test1': 'global-env2-default-1', 'test2': 'global-env2-default-2' } }, 'step-foo': [ { 'implementer': 'foo1', 'config': { 'test1': 'foo' } } ] } }) step_config = config.get_step_config('step-foo') sub_step = step_config.get_sub_step('foo1') self.assertEqual( ConfigValue.convert_leaves_to_values( sub_step.get_global_environment_defaults('env1') ), { 'environment-name': 'env1', 'test1': 'global-env1-default-1', 'test2': 'global-env1-default-2' } ) self.assertEqual( ConfigValue.convert_leaves_to_values( sub_step.get_global_environment_defaults('env2') ), { 'environment-name': 'env2', 'test1': 'global-env2-default-1', 'test2': 'global-env2-default-2' } )
def test_register_obfuscation_stream(self): secret_value = "decrypt me" config_value = ConfigValue(f'TEST_ENC[{secret_value}]') DecryptionUtils.register_config_value_decryptor( SampleConfigValueDecryptor()) out = io.StringIO() with redirect_stdout(out): old_stdout = sys.stdout new_stdout = TextIOSelectiveObfuscator(old_stdout) DecryptionUtils.register_obfuscation_stream(new_stdout) try: sys.stdout = new_stdout DecryptionUtils.decrypt(config_value) print( f"ensure that I can't actually leak secret value ({secret_value})" ) self.assertRegex( out.getvalue(), r"ensure that I can't actually leak secret value \(\*+\)") finally: new_stdout.close() sys.stdout = old_stdout
def test_value_decyrpt(self, sops_mock): encrypted_config_file_path = os.path.join( os.path.dirname(__file__), 'decryptors', 'files', 'step-runner-config-secret-stuff.yml') config_value = ConfigValue( value= 'ENC[AES256_GCM,data:UGKfnzsSrciR7GXZJhOCMmFrz3Y6V3pZsd3P,iv:yuReqA+n+rRXVHMc+2US5t7yPx54sooZSXWV4KLjDIs=,tag:jueP7/ZWLfYrEuhh+4eS8g==,type:str]', parent_source=encrypted_config_file_path, path_parts=[ 'step-runner-config', 'global-environment-defaults', 'DEV', 'kube-api-token' ]) DecryptionUtils.register_config_value_decryptor(SOPS()) sops_mock.side_effect = create_sops_side_effect('mock decrypted value') decrypted_value = config_value.value sops_mock.assert_called_once_with( '--decrypt', '--extract=["step-runner-config"]["global-environment-defaults"]["DEV"]["kube-api-token"]', None, encrypted_config_file_path, _in=None, _out=Any(StringIO), _err=Any(StringIO)) self.assertEqual(decrypted_value, 'mock decrypted value')
def test_decrypt_no_valid_key(self): encrypted_config_file_path = os.path.join( os.path.dirname(__file__), 'files', 'step-runner-config-secret-stuff.yml') config_value = ConfigValue( value= 'ENC[AES256_GCM,data:UGKfnzsSrciR7GXZJhOCMmFrz3Y6V3pZsd3P,iv:yuReqA+n+rRXVHMc+2US5t7yPx54sooZSXWV4KLjDIs=,tag:jueP7/ZWLfYrEuhh+4eS8g==,type:str]', parent_source=encrypted_config_file_path, path_parts=[ 'step-runner-config', 'global-environment-defaults', 'DEV', 'kube-api-token' ]) sops_decryptor = SOPS() # delete the gpg key needed to decrypt the value self.delete_gpg_key() # attempt to decrypt the value with self.assertRaisesRegex( RuntimeError, r"Error invoking sops when trying to decrypt config value \(ConfigValue\(.*\)\):" ): sops_decryptor.decrypt(config_value)
def test_get_sub_step_env_config(self): config = Config({ Config.CONFIG_KEY: { 'step-foo': [ { 'implementer': 'foo1', 'config': { 'test1': 'foo' }, 'environment-config': { 'env1': { 'test2': 'bar' } } } ] } }) step_config = config.get_step_config('step-foo') sub_step = step_config.get_sub_step('foo1') self.assertEqual( ConfigValue.convert_leaves_to_values( sub_step.get_sub_step_env_config('env1') ), { 'test2': 'bar' } )
def test_decrypt_parent_source_dict(self, sops_mock): encrypted_config_file_path = os.path.join( os.path.dirname(__file__), 'files', 'step-runner-config-secret-stuff.yml') encrypted_config = parse_yaml_or_json_file(encrypted_config_file_path) encrypted_config_json = json.dumps(encrypted_config) config_value = ConfigValue( value= 'ENC[AES256_GCM,data:UGKfnzsSrciR7GXZJhOCMmFrz3Y6V3pZsd3P,iv:yuReqA+n+rRXVHMc+2US5t7yPx54sooZSXWV4KLjDIs=,tag:jueP7/ZWLfYrEuhh+4eS8g==,type:str]', parent_source=encrypted_config, path_parts=[ 'step-runner-config', 'global-environment-defaults', 'DEV', 'kube-api-token' ]) sops_decryptor = SOPS() sops_decryptor.decrypt(config_value) sops_mock.assert_called_once_with( '--decrypt', '--extract=["step-runner-config"]["global-environment-defaults"]["DEV"]["kube-api-token"]', '--input-type=json', '/dev/stdin', _in=encrypted_config_json, _out=Any(StringIO), _err=Any(StringIO))
def test_sub_step_with_no_environment_config(self): config = Config({ Config.CONFIG_KEY: { 'step-foo': [ { 'implementer': 'foo1', 'config': { 'test1': 'foo' } } ] } }) step_config = config.get_step_config('step-foo') self.assertEqual(len(step_config.sub_steps), 1) self.assertEqual( ConfigValue.convert_leaves_to_values( step_config.get_sub_step('foo1').sub_step_config ), { 'test1': 'foo' } ) self.assertEqual(step_config.get_sub_step('foo1').get_sub_step_env_config('env1'), {})
def test_decrypt_additional_sops_args(self, sops_mock): encrypted_config_file_path = os.path.join( os.path.dirname(__file__), 'files', 'step-runner-config-secret-stuff.yml') config_value = ConfigValue( value= 'ENC[AES256_GCM,data:UGKfnzsSrciR7GXZJhOCMmFrz3Y6V3pZsd3P,iv:yuReqA+n+rRXVHMc+2US5t7yPx54sooZSXWV4KLjDIs=,tag:jueP7/ZWLfYrEuhh+4eS8g==,type:str]', parent_source=encrypted_config_file_path, path_parts=[ 'step-runner-config', 'global-environment-defaults', 'DEV', 'kube-api-token' ]) sops_decryptor = SOPS(additional_sops_args=['--aws-profile=foo']) sops_decryptor.decrypt(config_value) sops_mock.assert_called_once_with( '--decrypt', '--extract=["step-runner-config"]["global-environment-defaults"]["DEV"]["kube-api-token"]', None, encrypted_config_file_path, '--aws-profile=foo', _in=None, _out=Any(StringIO), _err=Any(StringIO))
def test_sub_step_with_name(self): config = Config({ Config.CONFIG_KEY: { 'step-foo': [ { 'name': 'sub-step-name-test', 'implementer': 'foo1', 'config': { 'test1': 'foo' } } ] } }) step_config = config.get_step_config('step-foo') self.assertEqual(len(step_config.sub_steps), 1) self.assertEqual( ConfigValue.convert_leaves_to_values( step_config.get_sub_step('sub-step-name-test').sub_step_config ), { 'test1': 'foo' } )
def test_add_or_update_sub_step_config_exising_sub_step(self): config = Config([ { Config.CONFIG_KEY: { 'step-foo': [{ 'implementer': 'foo1', 'config': { 'test1': 'foo' } }] } }, { Config.CONFIG_KEY: { 'step-foo': [{ 'implementer': 'foo1', 'config': { 'test2': 'foo' } }] } }, ]) step_config = config.get_step_config('step-foo') self.assertEqual(len(step_config.sub_steps), 1) self.assertEqual( ConfigValue.convert_leaves_to_values( step_config.get_sub_step('foo1').sub_step_config), { 'test1': 'foo', 'test2': 'foo' })
def test_sub_step_with_continue_sub_steps_on_failure_str(self): config = Config({ Config.CONFIG_KEY: { 'step-foo': [ { 'implementer': 'foo1', 'continue-sub-steps-on-failure': 'true', 'config': { 'test1': 'foo' } }, { 'implementer': 'foo2', 'config': { 'test2': 'foo' } } ] } }) step_config = config.get_step_config('step-foo') self.assertEqual(len(step_config.sub_steps), 2) self.assertEqual( ConfigValue.convert_leaves_to_values( step_config.get_sub_step('foo1').sub_step_config, ), { 'test1': 'foo' } ) self.assertEqual( ConfigValue.convert_leaves_to_values( step_config.get_sub_step('foo2').sub_step_config ), { 'test2': 'foo' } ) self.assertTrue( step_config.get_sub_step('foo1').sub_step_contine_sub_steps_on_failure ) self.assertFalse( step_config.get_sub_step('foo2').sub_step_contine_sub_steps_on_failure )
def _run_step(self): step_result = StepResult.from_step_implementer(self) runtime_step_config = self.config.get_copy_of_runtime_step_config( self.environment, self.step_implementer_config_defaults()) for n, v in ConfigValue.convert_leaves_to_values( runtime_step_config).items(): step_result.add_artifact(name=n, value=v) return step_result
def test_decrypt_sample_decryptor_does_not_match(self): config_value = ConfigValue('attempt to decrypt me') DecryptionUtils.register_config_value_decryptor( SampleConfigValueDecryptor()) decrypted_value = DecryptionUtils.decrypt(config_value) self.assertIsNone(decrypted_value)
def test_create_and_register_config_value_decryptor_no_constructor_args( self): DecryptionUtils.create_and_register_config_value_decryptor( 'tests.test_decryption_utils.SampleConfigValueDecryptor') secret_value = "decrypt me" config_value = ConfigValue(f'TEST_ENC[{secret_value}]') decrypted_value = DecryptionUtils.decrypt(config_value) self.assertEqual(decrypted_value, secret_value)
def test_decrypt_sample_decryptor(self): secret_value = "decrypt me" config_value = ConfigValue(f'TEST_ENC[{secret_value}]') DecryptionUtils.register_config_value_decryptor( SampleConfigValueDecryptor()) decrypted_value = DecryptionUtils.decrypt(config_value) self.assertEqual(decrypted_value, secret_value)
def test_create_and_register_config_value_decryptor_required_constructor_args( self): DecryptionUtils.create_and_register_config_value_decryptor( 'tests.test_decryption_utils.RequiredConstructorParamsConfigValueDecryptor', {'required_arg': 'hello world'}) secret_value = "decrypt me" config_value = ConfigValue(f'TEST_ENC[{secret_value}]') decrypted_value = DecryptionUtils.decrypt(config_value) self.assertEqual(decrypted_value, secret_value)
def test_value_path_given_no_inital_value_path_parts(self): source = { Config.CONFIG_KEY: { 'step-foo': [{ 'implementer': 'foo1', 'config': { 'test1': 'foo' } }] } } ConfigValue.convert_leaves_to_config_values(values=source, parent_source=source) self.assertEqual( source[Config.CONFIG_KEY]['step-foo'][0]['config'] ['test1'].path_parts, ['step-runner-config', 'step-foo', 0, 'config', 'test1'])
def maven_settings_file(self): """Gets the maven settings file for this step. """ if not self.__maven_settings_file: maven_servers = ConfigValue.convert_leaves_to_values( self.get_value('maven-servers')) maven_repositories = ConfigValue.convert_leaves_to_values( self.get_value('maven-repositories')) maven_mirrors = ConfigValue.convert_leaves_to_values( self.get_value('maven-mirrors')) self.__maven_settings_file = generate_maven_settings( working_dir=self.work_dir_path, maven_servers=maven_servers, maven_repositories=maven_repositories, maven_mirrors=maven_mirrors) return self.__maven_settings_file
def _run_step(self): step_result = StepResult.from_step_implementer(self) runtime_step_config = self.config.get_copy_of_runtime_step_config( self.environment, self.step_implementer_config_defaults()) # copy the key/value pairs into the artifacts for name, value in ConfigValue.convert_leaves_to_values( runtime_step_config).items(): # print(name, value) step_result.add_artifact(name, value) return step_result
def test__repr__(self): source = { Config.CONFIG_KEY: { 'step-foo': [{ 'implementer': 'foo1', 'config': { 'test1': 'foo' } }] } } ConfigValue.convert_leaves_to_config_values( values=source[Config.CONFIG_KEY], parent_source=source, path_parts=[Config.CONFIG_KEY]) self.assertEqual( str(source[Config.CONFIG_KEY]['step-foo'][0]['config']['test1']), "ConfigValue(value=foo, value_path='['step-runner-config', 'step-foo', 0, 'config', 'test1']')" )
def test_multiple_sub_steps(self): config = Config({ Config.CONFIG_KEY: { 'step-foo': [ { 'implementer': 'foo1', 'config': { 'test1': 'foo' } }, { 'implementer': 'foo2', 'config': { 'test2': 'foo' } } ] } }) step_config = config.get_step_config('step-foo') self.assertEqual(len(step_config.sub_steps), 2) self.assertEqual( ConfigValue.convert_leaves_to_values( step_config.get_sub_step('foo1').sub_step_config, ), { 'test1': 'foo' } ) self.assertEqual( ConfigValue.convert_leaves_to_values( step_config.get_sub_step('foo2').sub_step_config ), { 'test2': 'foo' } )
def test_get_sops_value_path(self, sops_mock): config_value = ConfigValue( value= 'ENC[AES256_GCM,data:UGKfnzsSrciR7GXZJhOCMmFrz3Y6V3pZsd3P,iv:yuReqA+n+rRXVHMc+2US5t7yPx54sooZSXWV4KLjDIs=,tag:jueP7/ZWLfYrEuhh+4eS8g==,type:str]', parent_source=None, path_parts=[ "step-runner-config", "step-foo", 0, "config", "test1" ]) sops_value_path = SOPS.get_sops_value_path(config_value) self.assertEqual( sops_value_path, '["step-runner-config"]["step-foo"][0]["config"]["test1"]')
def test_merge_valid_global_environment_defaults_same_env_diff_keys(self): with TempDirectory() as temp_dir: config_dir = "test" config_files = [ { 'name': os.path.join(config_dir,'foo.yml'), 'contents' : { Config.CONFIG_KEY: { 'global-environment-defaults' : { 'env1' : { 'foo-key': 'foo' } } } } }, { 'name': os.path.join(config_dir,'bar.yml'), 'contents' : { Config.CONFIG_KEY: { 'global-environment-defaults' : { 'env1' : { 'bar-key': 'bar' } } } } }, ] for config_file in config_files: config_file_name = config_file['name'] config_file_contents = config_file['contents'] temp_dir.write( config_file_name, bytes(f"{config_file_contents}", 'utf-8') ) config = Config() config.add_config(os.path.join(temp_dir.path, config_dir)) self.assertEqual( ConfigValue.convert_leaves_to_values( config.get_global_environment_defaults_for_environment('env1') ), { 'environment-name': 'env1', 'foo-key': 'foo', 'bar-key': 'bar' } )
def test_can_can_decrypt_not_string(self, sops_mock): encrypted_config_file_path = os.path.join( os.path.dirname(__file__), 'files', 'step-runner-config-secret-stuff.yml') config_value = ConfigValue(value=True, parent_source=encrypted_config_file_path, path_parts=[ 'step-runner-config', 'global-environment-defaults', 'DEV', 'kube-api-token' ]) sops_decryptor = SOPS() self.assertFalse(sops_decryptor.can_decrypt(config_value))
def test_can_decrypt_true(self): encrypted_config_file_path = os.path.join( os.path.dirname(__file__), 'files', 'step-runner-config-secret-stuff.yml') config_value = ConfigValue( value= 'ENC[AES256_GCM,data:UGKfnzsSrciR7GXZJhOCMmFrz3Y6V3pZsd3P,iv:yuReqA+n+rRXVHMc+2US5t7yPx54sooZSXWV4KLjDIs=,tag:jueP7/ZWLfYrEuhh+4eS8g==,type:str]', parent_source=encrypted_config_file_path, path_parts=[ 'step-runner-config', 'global-environment-defaults', 'DEV', 'kube-api-token' ]) sops_decryptor = SOPS() self.assertTrue(sops_decryptor.can_decrypt(config_value))