def test_get_sub_step_env_config(self):
config = Config({
Config.CONFIG_KEY: {
'step-foo': [
{
'implementer': 'foo1',
'config': {
'test1': 'foo'
},
'environment-config': {
'env1': {
'test2': 'bar'
}
}
}
]
}
})
step_config = config.get_step_config('step-foo')
sub_step = step_config.get_sub_step('foo1')
self.assertEqual(
ConfigValue.convert_leaves_to_values(
sub_step.get_sub_step_env_config('env1')
),
{
'test2': 'bar'
}
)
def test_merge_sub_step_config_duplicate_sub_step_environment_config_keys(
self):
with self.assertRaisesRegex(
ValueError,
r"Error merging new sub step environment configuration into existing sub step environment configuration for sub step \(foo1\) of step \(step-foo\): Conflict at env1.dup-key"
):
Config([{
Config.CONFIG_KEY: {
'step-foo': [{
'implementer': 'foo1',
'config': {
'test1': 'foo'
},
'environment-config': {
'env1': {
'dup-key': 'value1'
}
}
}]
}
}, {
Config.CONFIG_KEY: {
'step-foo': [{
'implementer': 'foo1',
'config': {
'test2': 'bar'
},
'environment-config': {
'env1': {
'dup-key': 'value2'
}
}
}]
}
}])
def test_initial_config_dict_valid_with_decryptor_definition(self):
config = Config({
'step-runner-config': {
'config-decryptors': [
{
'implementer': 'SOPS',
'config': {
'additional_sops_args': [
'--aws-profile=foo'
]
}
}
]
}
})
self.assertEqual(config.global_defaults, {})
self.assertEqual(config.global_environment_defaults, {})
sops_decryptor = DecryptionUtils._DecryptionUtils__config_value_decryptors[0]
self.assertEqual(
type(sops_decryptor),
SOPS
)
self.assertEqual(
sops_decryptor._SOPS__additional_sops_args,
['--aws-profile=foo']
)
def test_add_config_dict_missing_config_key(self):
config = Config()
with self.assertRaisesRegex(
AssertionError,
r"Failed to add invalid config. Missing expected top level key \(step-runner-config\):"
):
config.add_config({'foo': 'foo'})
def test_global_defaults(self):
config = Config({
Config.CONFIG_KEY: {
'global-defaults': {
'test1': 'global-default-1',
'test2': 'global-default-2'
},
'step-foo': [{
'implementer': 'foo1',
'config': {
'test1': 'foo'
}
}]
}
})
step_config = config.get_step_config('step-foo')
sub_step = step_config.get_sub_step('foo1')
self.assertEqual(
sub_step.global_defaults, {
'test1':
ConfigValue(
'global-default-1', None,
["step-runner-config", "global-defaults", "test1"]),
'test2':
ConfigValue('global-default-2', None,
["step-runner-config", "global-defaults", "test2"])
})
def test_write_working_file(self):
config = Config({
'step-runner-config': {
'foo': {
'implementer':
'tests.helpers.sample_step_implementers.FooStepImplementer',
'config': {}
}
}
})
step_config = config.get_step_config('foo')
sub_step = step_config.get_sub_step(
'tests.helpers.sample_step_implementers.FooStepImplementer')
with TempDirectory() as test_dir:
results_dir_path = os.path.join(test_dir.path,
'step-runner-results')
working_dir_path = os.path.join(test_dir.path,
'step-runner-working')
step = FooStepImplementer(
results_dir_path=results_dir_path,
results_file_name='step-runner-results.yml',
work_dir_path=working_dir_path,
config=sub_step)
step.write_working_file('test-working-file', b'hello world')
with open(os.path.join(working_dir_path, 'foo', 'test-working-file'), 'r') \
as working_file:
self.assertEqual(working_file.read(), 'hello world')
def test_add_config_invalid_type(self):
config = Config()
with self.assertRaisesRegex(
ValueError,
r"Given config \(True\) is unexpected type \(<class 'bool'>\) not a dictionary, string, or list of former."
):
config.add_config(True)
def test_create_working_dir_sub_dir(self):
config = Config({
'step-runner-config': {
'foo': {
'implementer':
'tests.helpers.sample_step_implementers.FooStepImplementer',
'config': {}
}
}
})
step_config = config.get_step_config('foo')
sub_step = step_config.get_sub_step(
'tests.helpers.sample_step_implementers.FooStepImplementer')
with TempDirectory() as test_dir:
working_dir_path = os.path.join(test_dir.path,
'step-runner-working')
step = FooStepImplementer(workflow_result=WorkflowResult(),
parent_work_dir_path=working_dir_path,
config=sub_step)
sub = step.create_working_dir_sub_dir('folder1')
self.assertEqual(sub, f'{working_dir_path}/foo/folder1')
self.assertEqual(True, os.path.isdir(sub))
sub = step.create_working_dir_sub_dir('folder1/folder2/folder3')
self.assertEqual(
sub, f'{working_dir_path}/foo/folder1/folder2/folder3')
self.assertEqual(True, os.path.isdir(sub))
def test_set_step_config_override_overrides_existing_step(self):
config = Config({
Config.CONFIG_KEY: {
'step-foo': {
'implementer': 'foo',
}
}
})
sub_step_configs = config.get_sub_step_configs('step-foo')
self.assertEqual(len(sub_step_configs), 1)
sub_step_config = sub_step_configs[0]
self.assertEqual(sub_step_config.step_config_overrides, {})
config.set_step_config_overrides('step-foo', {
'test1': 'test2'
})
self.assertEqual(sub_step_config.step_config_overrides, {
'test1': 'test2'
})
config.set_step_config_overrides('step-foo', {
'test2': 'test3'
})
self.assertEqual(sub_step_config.step_config_overrides, {
'test2': 'test3'
})
def test_from_step_implementer_with_env(self):
config = Config({
'step-runner-config': {
'foo': {
'implementer':
'tests.helpers.sample_step_implementers.FooStepImplementer',
'config': {}
}
}
})
step_config = config.get_step_config('foo')
sub_step = step_config.get_sub_step(
'tests.helpers.sample_step_implementers.FooStepImplementer')
step = FooStepImplementer(workflow_result=WorkflowResult(),
parent_work_dir_path=None,
config=sub_step,
environment='blarg')
step_result = StepResult.from_step_implementer(step)
expected_step_result = StepResult(
step_name='foo',
sub_step_name=
'tests.helpers.sample_step_implementers.FooStepImplementer',
sub_step_implementer_name=
'tests.helpers.sample_step_implementers.FooStepImplementer',
environment='blarg')
self.assertEqual(step_result, expected_step_result)
def test_sub_step_with_name(self):
config = Config({
Config.CONFIG_KEY: {
'step-foo': [
{
'name': 'sub-step-name-test',
'implementer': 'foo1',
'config': {
'test1': 'foo'
}
}
]
}
})
step_config = config.get_step_config('step-foo')
self.assertEqual(len(step_config.sub_steps), 1)
self.assertEqual(
ConfigValue.convert_leaves_to_values(
step_config.get_sub_step('sub-step-name-test').sub_step_config
),
{
'test1': 'foo'
}
)
def test_initial_config_dict_valid_basic(self):
config = Config({
Config.CONFIG_KEY: {}
})
self.assertEqual(config.global_defaults, {})
self.assertEqual(config.global_environment_defaults, {})
def test_sub_step_with_continue_sub_steps_on_failure_str(self):
config = Config({
Config.CONFIG_KEY: {
'step-foo': [{
'implementer': 'foo1',
'continue-sub-steps-on-failure': 'true',
'config': {
'test1': 'foo'
}
}, {
'implementer': 'foo2',
'config': {
'test2': 'foo'
}
}]
}
})
step_config = config.get_step_config('step-foo')
self.assertEqual(len(step_config.sub_steps), 2)
self.assertEqual(
ConfigValue.convert_leaves_to_values(
step_config.get_sub_step('foo1').sub_step_config, ),
{'test1': 'foo'})
self.assertEqual(
ConfigValue.convert_leaves_to_values(
step_config.get_sub_step('foo2').sub_step_config),
{'test2': 'foo'})
self.assertTrue(
step_config.get_sub_step(
'foo1').sub_step_contine_sub_steps_on_failure)
self.assertFalse(
step_config.get_sub_step(
'foo2').sub_step_contine_sub_steps_on_failure)
def test_get_config_value_list(self):
config = Config({
Config.CONFIG_KEY: {
'global-defaults': {
'global-default-unique-0':
['global-default-a', 'global-default-b'],
},
'step-foo': [{
'implementer': 'foo1',
'config': {
'step-foo-foo1-unique-0':
['step-foo-foo1-a', 'step-foo-foo1-b'],
}
}]
}
})
step_config = config.get_step_config('step-foo')
sub_step = step_config.get_sub_step('foo1')
self.assertEqual(sub_step.get_config_value('global-default-unique-0'),
['global-default-a', 'global-default-b'])
self.assertEqual(sub_step.get_config_value('step-foo-foo1-unique-0'),
['step-foo-foo1-a', 'step-foo-foo1-b'])
def test_merge_sub_step_config_duplicate_sub_step_keys(self):
with self.assertRaisesRegex(
ValueError,
r"Error merging new sub step configuration into existing sub step configuration for sub step \(foo1\) of step \(step-foo\): Conflict at test1"
):
Config([{
Config.CONFIG_KEY: {
'step-foo': [
{
'implementer': 'foo1',
'config': {
'test1': 'foo'
}
},
]
}
}, {
Config.CONFIG_KEY: {
'step-foo': [
{
'implementer': 'foo1',
'config': {
'test1': 'bar'
}
},
]
}
}])
def create_given_step_implementer_rekor_sign_generic(
self,
step_implementer,
step_config={},
step_name='',
environment=None,
implementer='',
workflow_result=None,
parent_work_dir_path='',
artifact_to_sign_uri_config_key=None):
config = Config({
Config.CONFIG_KEY: {
step_name: [{
'implementer': implementer,
'config': step_config
}]
}
})
step_config = config.get_step_config(step_name)
sub_step_config = step_config.get_sub_step(implementer)
if not workflow_result:
workflow_result = WorkflowResult()
step_implementer = step_implementer(
workflow_result=workflow_result,
parent_work_dir_path=parent_work_dir_path,
config=sub_step_config,
artifact_to_sign_uri_config_key=artifact_to_sign_uri_config_key,
environment=environment)
return step_implementer
def test_add_or_update_sub_step_config_exising_sub_step(self):
config = Config([
{
Config.CONFIG_KEY: {
'step-foo': [{
'implementer': 'foo1',
'config': {
'test1': 'foo'
}
}]
}
},
{
Config.CONFIG_KEY: {
'step-foo': [{
'implementer': 'foo1',
'config': {
'test2': 'foo'
}
}]
}
},
])
step_config = config.get_step_config('step-foo')
self.assertEqual(len(step_config.sub_steps), 1)
self.assertEqual(
ConfigValue.convert_leaves_to_values(
step_config.get_sub_step('foo1').sub_step_config), {
'test1': 'foo',
'test2': 'foo'
})
def test_use_object_property_with_config_value(self):
workflow_result = WorkflowResult()
parent_work_dir_path = '/fake/path'
step_config = {
'tox-env': 'config-value-fake-env'
}
config = Config({
Config.CONFIG_KEY: {
'foo': [
{
'implementer': 'ToxGeneric',
'config': step_config
}
]
}
})
step_implementer = ToxGeneric(
workflow_result=workflow_result,
parent_work_dir_path=parent_work_dir_path,
config=config,
tox_env='object-property-fake-env'
)
self.assertEqual(
step_implementer.tox_env,
'object-property-fake-env'
)
def test_multiple_sub_steps(self):
config = Config({
Config.CONFIG_KEY: {
'step-foo': [{
'implementer': 'foo1',
'config': {
'test1': 'foo'
}
}, {
'implementer': 'foo2',
'config': {
'test2': 'foo'
}
}]
}
})
step_config = config.get_step_config('step-foo')
self.assertEqual(len(step_config.sub_steps), 2)
self.assertEqual(
ConfigValue.convert_leaves_to_values(
step_config.get_sub_step('foo1').sub_step_config, ),
{'test1': 'foo'})
self.assertEqual(
ConfigValue.convert_leaves_to_values(
step_config.get_sub_step('foo2').sub_step_config),
{'test2': 'foo'})
def test_sub_step_with_no_environment_config(self):
config = Config({
Config.CONFIG_KEY: {
'step-foo': [
{
'implementer': 'foo1',
'config': {
'test1': 'foo'
}
}
]
}
})
step_config = config.get_step_config('step-foo')
self.assertEqual(len(step_config.sub_steps), 1)
self.assertEqual(
ConfigValue.convert_leaves_to_values(
step_config.get_sub_step('foo1').sub_step_config
),
{
'test1': 'foo'
}
)
self.assertEqual(step_config.get_sub_step('foo1').get_sub_step_env_config('env1'), {})
def test_add_config_dir_one_valid_file(self):
with TempDirectory() as temp_dir:
config_dir = "test"
config_files = [
{
'name': os.path.join(config_dir,'foo.yml'),
'contents' : {
Config.CONFIG_KEY: {}
}
}
]
for config_file in config_files:
config_file_name = config_file['name']
config_file_contents = config_file['contents']
temp_dir.write(
config_file_name,
bytes(f"{config_file_contents}", 'utf-8')
)
config = Config()
config.add_config(os.path.join(temp_dir.path, config_dir))
self.assertEqual(config.global_defaults, {})
self.assertEqual(config.global_environment_defaults, {})
def test_fail(self):
config = Config({
'step-runner-config': {
'foo': {
'implementer':
'tests.helpers.sample_step_implementers.FailStepImplementer',
'config': {}
}
}
})
step_config = config.get_step_config('foo')
sub_step = step_config.get_sub_step(
'tests.helpers.sample_step_implementers.FailStepImplementer')
with TempDirectory() as test_dir:
results_dir_path = os.path.join(test_dir.path,
'step-runner-results')
working_dir_path = os.path.join(test_dir.path,
'step-runner-working')
step = FailStepImplementer(
results_dir_path=results_dir_path,
results_file_name='step-runner-results.yml',
work_dir_path=working_dir_path,
config=sub_step)
result = step.run_step()
self.assertEqual(False, result)
def test_add_config_dir_one_valid_file_one_invalid_file(self):
with TempDirectory() as temp_dir:
config_dir = "test"
config_files = [
{
'name': os.path.join(config_dir,'foo.yml'),
'contents' : {
Config.CONFIG_KEY: {}
}
},
{
'name': os.path.join(config_dir,'bad.yml'),
'contents' : {
'bad': {
'implementer': 'bar'
}
}
}
]
for config_file in config_files:
config_file_name = config_file['name']
config_file_contents = config_file['contents']
temp_dir.write(
config_file_name,
bytes(f"{config_file_contents}", 'utf-8')
)
config = Config()
with self.assertRaisesRegex(
AssertionError,
r"Failed to add parsed configuration file \(.*\): Failed to add invalid config. Missing expected top level key \(step-runner-config\):"
):
config.add_config(os.path.join(temp_dir.path, config_dir))
def test_write_working_file_touch(self):
config = Config({
'step-runner-config': {
'foo': {
'implementer':
'tests.helpers.sample_step_implementers.FooStepImplementer',
'config': {}
}
}
})
step_config = config.get_step_config('foo')
sub_step = step_config.get_sub_step(
'tests.helpers.sample_step_implementers.FooStepImplementer')
with TempDirectory() as test_dir:
working_dir_path = os.path.join(test_dir.path,
'step-runner-working')
step = FooStepImplementer(workflow_result=WorkflowResult(),
parent_work_dir_path=working_dir_path,
config=sub_step)
step.write_working_file('foo/test.json')
working_file_path = os.path.join(working_dir_path,
'foo/foo/test.json')
self.assertTrue(os.path.exists(working_file_path))
with open(working_file_path, 'r') as working_file:
self.assertEqual(working_file.read(), '')
def test_add_two_valid_files(self):
with TempDirectory() as temp_dir:
config_dir = "test"
config_files = [
{
'name': os.path.join(config_dir,'foo.yml'),
'contents' : {
Config.CONFIG_KEY: {
'step-test-foo' : {
'implementer': 'foo',
'config': {
'f': 'hello1'
}
}
}
}
},
{
'name': os.path.join(config_dir,'bar.yml'),
'contents' : {
Config.CONFIG_KEY: {
'step-test-bar' : {
'implementer': 'bar',
'config': {
'b': 'hello2'
}
}
}
}
},
]
for config_file in config_files:
config_file_name = config_file['name']
config_file_contents = config_file['contents']
temp_dir.write(
config_file_name,
bytes(f"{config_file_contents}", 'utf-8')
)
config = Config()
config.add_config(
[
os.path.join(temp_dir.path, config_dir, 'foo.yml'),
os.path.join(temp_dir.path, config_dir, 'bar.yml')
]
)
self.assertEqual(config.global_defaults, {})
self.assertEqual(config.global_environment_defaults, {})
self.assertEqual(
config.get_step_config('step-test-foo').get_sub_step('foo').get_config_value('f'),
'hello1'
)
self.assertEqual(
config.get_step_config('step-test-bar').get_sub_step('bar').get_config_value('b'),
'hello2'
)
def test_add_config_file_missing_file(self):
with TempDirectory() as temp_dir:
config = Config()
with self.assertRaisesRegex(
ValueError,
r"Given config string \(.*\) is not a valid path."
):
config.add_config(os.path.join(temp_dir.path, 'does-not-exist.yml'))
def test_add_config_dir_no_files(self):
with TempDirectory() as temp_dir:
config = Config()
with self.assertRaisesRegex(
ValueError,
r"Given config string \(.*\) is a directory with no recursive children files."
):
config.add_config(temp_dir.path)
def test_get_sub_step_configs_for_non_existent_step(self):
config = Config(
{Config.CONFIG_KEY: {
'step-foo': {
'implementer': 'foo'
}
}})
self.assertEqual(config.get_sub_step_configs('does-not-exist'), [])
def test_invalid_sub_steps(self):
with self.assertRaisesRegex(
ValueError,
r"Expected step \(step-foo\) to have have step config " \
r"\(ConfigValue\(value=bad-step-config, " \
r"value_path='\['step-runner-config', 'step-foo'\]\'\)\) of " \
r"type dict or list but got: <class 'ploigos_step_runner.config.config_value.ConfigValue'>"
):
Config({Config.CONFIG_KEY: {'step-foo': "bad-step-config"}})
def test_set_step_config_overrides_not_existing_step(self):
config = Config({Config.CONFIG_KEY: {}})
self.assertIsNone(config.get_step_config('step-bar'))
config.set_step_config_overrides('step-bar', {'test1': 'test2'})
step_config = config.get_step_config('step-bar')
self.assertIsNotNone(step_config)
self.assertEqual(step_config.step_config_overrides, {'test1': 'test2'})
