def test_c4_orchestration_manager_find_stack():
mocked_boto3 = MockBoto3()
mock_stack_names = [
'foo-bar-baz', 'awseb-e-foo-stack', 'c4-network-cgap-test-123',
'c4-iam-cgap-test-456'
]
MockBotoCloudFormationClient.setup_boto3_mocked_stacks(
boto3=mocked_boto3, mock_stack_names=mock_stack_names)
with mock.patch.object(cloudformation_utils, "boto3", mocked_boto3):
manager = cloudformation_utils.C4OrchestrationManager()
network_stack = manager.find_stack('network')
assert network_stack.name == 'c4-network-cgap-test-123'
mocked_boto3 = MockBoto3()
mock_stack_names = [
'foo-bar-baz', 'awseb-e-foo-stack', 'c4-network-cgap-test-123',
'c4-network-cgap-prod-456'
]
MockBotoCloudFormationClient.setup_boto3_mocked_stacks(
boto3=mocked_boto3, mock_stack_names=mock_stack_names)
with mock.patch.object(cloudformation_utils, "boto3", mocked_boto3):
manager = cloudformation_utils.C4OrchestrationManager()
with pytest.raises(ValueError, match="too many"):
manager.find_stack('network')
def test_get_ecs_real_url():
mocked_boto3 = MockBoto3()
mastertest_url = 'http://C4EcsTrialAlphacgapmastertest-1234512345.us-east-1.elb.amazonaws.com'
mastertest_outputs = [
{
'OutputKey': 'ECSApplicationURLcgapmastertest',
'OutputValue': mastertest_url,
'Description': 'URL of CGAP-Portal.',
},
]
supertest_url = 'http://cgap-supertest-11223344.us-east-1.elb.amazonaws.com'
supertest_outputs = [{
'OutputKey': 'ECSApplicationURLcgapsupertest',
'OutputValue': supertest_url,
'Description': 'URL of CGAP-Portal.',
}]
mocked_stacks = [
MockBotoCloudFormationStack('c4-iam-main-stack'),
MockBotoCloudFormationStack('c4-network-main-stack'),
MockBotoCloudFormationStack('c4-ecs-cgap-supertest-stack',
mock_outputs=supertest_outputs),
MockBotoCloudFormationStack('c4-ecs-cgap-mastertest-stack',
mock_outputs=mastertest_outputs),
]
MockBotoCloudFormationClient.setup_boto3_mocked_stacks(
boto3=mocked_boto3, mocked_stacks=mocked_stacks)
with mock.patch.object(cloudformation_utils, "boto3", mocked_boto3):
assert cloudformation_utils.get_ecs_real_url(
'cgap-supertest') == supertest_url
assert cloudformation_utils.get_ecs_real_url(
'cgap-mastertest') == mastertest_url
def mocked_s3_integration(integrated_names=None, zip_suffix="", ffenv=None):
"""
This does common setup of some mocks needed by zip testing.
"""
zip_path_key = "zip_path" + zip_suffix
zip_filename_key = "zip_filename" + zip_suffix
b3 = MockBoto3()
if not ffenv:
ffenv = integrated_names['ffenv'] if integrated_names else None
with mock.patch.object(s3_utils_module, "boto3", b3):
s3_connection = s3Utils(env=ffenv)
if integrated_names is not None:
# Not needed when mocked.
# s3_connection.s3_delete_dir(prefix)
# In our mock, this won't exist already on S3 like in the integrated version of this test,
# so we have to pre-load to our mock S3 manually. -kmp 13-Jan-2021
s3_connection.s3.upload_file(Filename=integrated_names[zip_path_key],
Bucket=s3_connection.outfile_bucket,
Key=integrated_names[zip_filename_key])
s3_connection.s3.put_object(Bucket=s3_connection.outfile_bucket,
Key=integrated_names['filename'],
Body=str.encode('thisisatest'))
yield s3_connection
def boto3_for_some_secrets_testing():
mocked_boto3 = MockBoto3()
manager = mocked_boto3.client('secretsmanager')
manager.put_secret_value_for_testing(SecretId=some_secret_identity,
Value=some_secret_string)
manager.put_secret_value_for_testing(SecretId=decoy_1_identity,
Value=decoy_1_string)
manager.put_secret_value_for_testing(SecretId=decoy_2_identity,
Value=decoy_2_string)
return mocked_boto3
def test_c4_orchestration_manager_find_stack_resource():
mocked_boto3 = MockBoto3()
private_subnet_a = MockBotoCloudFormationResourceSummary(
logical_id='MyPrivateSubnetA', physical_resource_id='subnet-111111')
private_subnet_b = MockBotoCloudFormationResourceSummary(
logical_id='MyPrivateSubnetB', physical_resource_id='subnet-222222')
public_subnet_a = MockBotoCloudFormationResourceSummary(
logical_id='MyPublicSubnetA', physical_resource_id='subnet-333333')
public_subnet_b = MockBotoCloudFormationResourceSummary(
logical_id='MyPublicSubnetA', physical_resource_id='subnet-444444')
network_resource_summaries = [
private_subnet_a, private_subnet_b, public_subnet_a, public_subnet_b
]
mocked_stacks = [
MockBotoCloudFormationStack(
'c4-network-main-stack',
mock_resource_summaries=network_resource_summaries),
MockBotoCloudFormationStack('c4-iam-main-stack')
]
MockBotoCloudFormationClient.setup_boto3_mocked_stacks(
boto3=mocked_boto3, mocked_stacks=mocked_stacks)
with mock.patch.object(cloudformation_utils, "boto3", mocked_boto3):
manager = cloudformation_utils.C4OrchestrationManager()
assert manager.find_stack_resource(
'network', 'MyPrivateSubnetA') == private_subnet_a
assert manager.find_stack_resource(
'network', 'MyPrivateSubnetB') == private_subnet_b
assert manager.find_stack_resource('network',
'MyPrivateSubnetC') is None
assert manager.find_stack_resource('network', 'MyPrivateSubnetA',
'logical_id') == 'MyPrivateSubnetA'
assert manager.find_stack_resource('network', 'MyPrivateSubnetC',
'logical_id') is None
assert manager.find_stack_resource(
'network', 'MyPrivateSubnetA',
'physical_resource_id') == 'subnet-111111'
assert manager.find_stack_resource('network', 'MyPrivateSubnetC',
'physical_resource_id') is None
assert manager.find_stack_resource('network', 'MyPrivateSubnetC',
'physical_resource_id',
'foo') == 'foo'
def test_c4_orchestration_manager_find_stack_outputs():
mocked_boto3 = MockBoto3()
network_outputs = [
{
'OutputKey': 'rds',
'OutputValue': 'some-db-thing'
},
{
'OutputKey': 'es',
'OutputValue': 'some-es-thing'
},
]
iam_outputs = [
{
'OutputKey': 'user1',
'OutputValue': 'Joe'
},
{
'OutputKey': 'user2',
'OutputValue': 'Sally'
},
]
mocked_stacks = [
MockBotoCloudFormationStack('c4-network-main-stack',
mock_outputs=network_outputs),
MockBotoCloudFormationStack('c4-iam-main-stack',
mock_outputs=iam_outputs),
]
MockBotoCloudFormationClient.setup_boto3_mocked_stacks(
boto3=mocked_boto3, mocked_stacks=mocked_stacks)
with mock.patch.object(cloudformation_utils, "boto3", mocked_boto3):
manager = cloudformation_utils.C4OrchestrationManager()
# Looking up a simple key by name
assert manager.find_stack_outputs('user1') == {'user1': 'Joe'}
assert manager.find_stack_outputs('user1', value_only=True) == ['Joe']
assert manager.find_stack_outputs('rds') == {'rds': 'some-db-thing'}
assert manager.find_stack_outputs(
'rds', value_only=True) == ['some-db-thing']
assert manager.find_stack_outputs('not-there') == {}
assert manager.find_stack_outputs('not-there', value_only=True) == []
# Use of predicate to find several related keys
assert manager.find_stack_outputs(lambda x: x.startswith("user")) == {
'user1': 'Joe',
'user2': 'Sally'
}
assert sorted(
manager.find_stack_outputs(lambda x: x.startswith("user"),
value_only=True)) == ['Joe', 'Sally']
def test_get_ecr_repo_url_hybrid(use_ecosystem_repo):
main_repo = 'xxx6742.dkr.ecr.us-east-1.amazonaws.com/main'
main_stack = 'c4-ecr-main-stack'
main_outputs = [
{
'OutputKey': 'C4ECRMainRepoURL',
'OutputValue': main_repo,
'Description': 'CGAPDocker Image Repository URL',
'ExportName': f'{main_stack}-RepoURL'
},
]
mocked_boto3 = MockBoto3()
mastertest_repo = 'xxx6742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest'
mastertest_stack = 'c4-ecr-cgap-mastertest-stack'
mastertest_outputs = [
{
'OutputKey': 'C4EcrCgapMastertestRepoURL',
'OutputValue': mastertest_repo,
'Description': 'CGAPDocker Image Repository URL',
'ExportName': f'{mastertest_stack}-RepoURL'
},
]
hotseat_repo = 'xxx6742.dkr.ecr.us-east-1.amazonaws.com/cgap-hotseat'
hotseat_stack = 'c4-ecr-cgap-hotseat-stack'
hotseat_outputs = [
{
'OutputKey': 'C4EcrCgapHotseatRepoURL',
'OutputValue': hotseat_repo,
'Description': 'CGAPDocker Image Repository URL',
'ExportName': f'{hotseat_stack}-RepoURL'
},
]
mocked_stacks = [
MockBotoCloudFormationStack('c4-iam-trial-alpha-stack'),
MockBotoCloudFormationStack('c4-network-trial-alpha-stack'),
MockBotoCloudFormationStack(mastertest_stack,
mock_outputs=mastertest_outputs),
MockBotoCloudFormationStack(hotseat_stack,
mock_outputs=hotseat_outputs),
]
if use_ecosystem_repo:
mocked_stacks.append(
MockBotoCloudFormationStack(main_stack, mock_outputs=main_outputs))
MockBotoCloudFormationClient.setup_boto3_mocked_stacks(
boto3=mocked_boto3, mocked_stacks=mocked_stacks)
with mock.patch.object(cloudformation_utils, "boto3", mocked_boto3):
assert cloudformation_utils.get_ecr_repo_url(
'cgap-mastertest') == mastertest_repo
assert cloudformation_utils.get_ecr_repo_url(
env_name='cgap-hotseat') == hotseat_repo
assert cloudformation_utils.get_ecr_repo_url(
env_name='cgap-wolf') == (main_repo if use_ecosystem_repo else '')
def test_c4_orchestration_manager_all_stacks():
mocked_boto3 = MockBoto3()
mock_stack_names = [
'foo-bar-baz', 'awseb-e-123-stack', 'c4-foo-123', 'c4-foo-456'
]
MockBotoCloudFormationClient.setup_boto3_mocked_stacks(
boto3=mocked_boto3, mock_stack_names=mock_stack_names)
with mock.patch.object(cloudformation_utils, "boto3", mocked_boto3):
manager = cloudformation_utils.C4OrchestrationManager()
all_stacks = manager.all_stacks()
assert sorted(s.name
for s in all_stacks) == ['c4-foo-123', 'c4-foo-456']
def mocked_s3utils(beanstalks=None,
require_sse=False,
other_access_key_names=None):
"""
This context manager sets up a mock version of boto3 for use by s3_utils and ff_utils during the context
of its test. It also sets up the S3_ENCRYPT_KEY environment variable with a sample value for testing,
and it sets up a set of mocked beanstalks for fourfront-foo and fourfront-bar, so that s3_utils will not
get confused when it does discovery operations to find them.
"""
if beanstalks is None:
beanstalks = TestScenarios.DEFAULT_BEANSTALKS
# First we make a mocked boto3 that will use an S3 mock with mock server side encryption.
s3_class = (make_mock_boto_s3_with_sse(
beanstalks=beanstalks, other_access_key_names=other_access_key_names)
if require_sse else MockBotoS3Client)
mock_boto3 = MockBoto3(
s3=s3_class,
elasticbeanstalk=make_mock_boto_eb_client_class(beanstalks=beanstalks))
# Now we arrange that both s3_utils and ff_utils modules share the illusion that our mock IS the boto3 library
with mock.patch.object(s3_utils, "boto3", mock_boto3):
with mock.patch.object(ff_utils, "boto3", mock_boto3):
with mock.patch.object(beanstalk_utils, "boto3", mock_boto3):
with mock.patch.object(base, "boto3", mock_boto3):
with mock.patch.object(
s3_utils.EnvManager,
"fetch_health_page_json") as mock_fetcher:
# This is all that's needed for s3Utils to initialize an EnvManager.
# We might have to add more later.
def mocked_fetch_health_page_json(
url, use_urllib=True):
ignored(use_urllib) # we don't test this
m = re.match(r'.*(fourfront-[a-z0-9-]+)(?:[.]|$)',
url)
if m:
env_name = m.group(1)
return make_mock_health_page(env_name)
else:
raise NotImplementedError(
f"Mock can't handle URL: {url}")
mock_fetcher.side_effect = mocked_fetch_health_page_json
# The mocked encrypt key is expected by various tools in the s3_utils module to be supplied
# as an environment variable (i.e., in os.environ), so this sets up that environment variable.
if require_sse:
with override_environ(
S3_ENCRYPT_KEY=s3_class.SSE_ENCRYPT_KEY):
yield
else:
yield
def test_c4_orchestration_manager_extract_stack_name_token():
mocked_boto3 = MockBoto3()
with mock.patch.object(cloudformation_utils, "boto3", mocked_boto3):
def extract(stack_name):
stack = MockBotoCloudFormationStack(stack_name)
manager = cloudformation_utils.C4OrchestrationManager()
return manager._extract_stack_name_token(
stack) # noQA - I know this is protected. Just testing it.
assert extract('foo-bar') is None
assert extract('c4-foo') == 'foo'
assert extract('c4-foo-bar') == 'foo'
assert extract('c4-foo-bar-baz') == 'foo'
assert extract('c4-foo_bar-baz') == 'foo_bar'
def test_c4_orchestration_manager_get_stack_output():
mocked_boto3 = MockBoto3()
with mock.patch.object(cloudformation_utils, "boto3", mocked_boto3):
manager = cloudformation_utils.C4OrchestrationManager()
mock_outputs = [{
'OutputKey': 'alpha',
'OutputValue': 'one'
}, {
'OutputKey': 'beta',
'OutputValue': 'two'
}]
s = MockBotoCloudFormationStack('foo', mock_outputs=mock_outputs)
assert manager.get_stack_output(s, 'alpha') == 'one'
assert manager.get_stack_output(s, 'beta') == 'two'
assert manager.get_stack_output(s, 'gamma') is None
def test_awseb_orchestration_manager_extract_stack_name_token():
mocked_boto3 = MockBoto3()
with mock.patch.object(cloudformation_utils, "boto3", mocked_boto3):
def extract(stack_name):
stack = MockBotoCloudFormationStack(stack_name)
manager = cloudformation_utils.AwsebOrchestrationManager()
return manager._extract_stack_name_token(
stack
) # noQA - please don't tell me this is protected. Just testing
assert extract('foo-bar') is None
assert extract('c4-foo') is None
assert extract('awseb-foo-bar') is None
assert extract('awseb-x-bar') is None
assert extract('awseb-e-bar') is None
assert extract('awseb-e-bar-stack') == 'bar'
assert extract('awseb-e-bar-baz-stack') == 'bar-baz'
def test_get_ecr_repo_url_kmp(
): # Test for my test environment. -kmp 17-Aug-2021
mocked_boto3 = MockBoto3()
supertest_repo = 'xxx0312.dkr.ecr.us-east-1.amazonaws.com/main'
supertest_outputs = [{
'OutputKey': 'C4ECRMainRepoURL',
'OutputValue': supertest_repo,
'Description': 'CGAPDocker Image Repository URL',
'ExportName': 'c4-ecr-main-stack-RepoURL'
}]
mocked_stacks = [
MockBotoCloudFormationStack('c4-iam-trial-alpha-stack'),
MockBotoCloudFormationStack('c4-network-trial-alpha-stack'),
MockBotoCloudFormationStack('c4-ecr-trial-alpha-stack',
mock_outputs=supertest_outputs),
]
MockBotoCloudFormationClient.setup_boto3_mocked_stacks(
boto3=mocked_boto3, mocked_stacks=mocked_stacks)
with mock.patch.object(cloudformation_utils, "boto3", mocked_boto3):
assert cloudformation_utils.get_ecr_repo_url(
'cgap-supertest') == supertest_repo
def test_get_ecr_repo_url_alpha():
mocked_boto3 = MockBoto3()
mastertest_repo = 'xxx6742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest'
mastertest_outputs = [
{
'OutputKey': 'C4EcrTrialAlphaECRRepoURL',
'OutputValue': mastertest_repo,
'Description': 'CGAPDocker Image Repository URL',
'ExportName': 'c4-ecr-trial-alpha-stack-ECRRepoURL'
},
]
mocked_stacks = [
MockBotoCloudFormationStack('c4-iam-trial-alpha-stack'),
MockBotoCloudFormationStack('c4-network-trial-alpha-stack'),
MockBotoCloudFormationStack('c4-ecr-trial-alpha-stack',
mock_outputs=mastertest_outputs),
]
MockBotoCloudFormationClient.setup_boto3_mocked_stacks(
boto3=mocked_boto3, mocked_stacks=mocked_stacks)
with mock.patch.object(cloudformation_utils, "boto3", mocked_boto3):
assert cloudformation_utils.get_ecr_repo_url(
'cgap-mastertest') == mastertest_repo
def test_env_manager():
class MyS3(MockBotoS3Client):
MOCK_STATIC_FILES = {
# Bucket 'global-env-1'
'global-env-1/cgap-footest':
'{"fourfront": "http://portal", "es": "http://es", "ff_env": "cgap-footest"}',
# Bucket 'global-env-2'
'global-env-2/cgap-footest':
'{"fourfront": "http://portal-foo", "es": "http://es-foo", "ff_env": "cgap-footest"}',
'global-env-2/cgap-bartest':
'{"fourfront": "http://portal-bar", "es": "http://es-bar", "ff_env": "cgap-bartest"}',
}
with mock.patch.object(s3_utils_module, "boto3", MockBoto3(s3=MyS3)):
my_s3 = s3_utils_module.boto3.client('s3')
with EnvManager.global_env_bucket_named(name='global-env-1'):
e = EnvManager(s3=my_s3)
assert e.portal_url == "http://portal"
assert e.es_url == "http://es"
assert e.env_name == "cgap-footest"
def test_env_manager_verify_and_get_env_config():
class MyS3(MockBotoS3Client):
MOCK_STATIC_FILES = {
# Bucket 'global-env-1'
'global-env-1/cgap-footest':
'{"fourfront": "http://portal", "es": "http://es", "ff_env": "cgap-footest"}',
# Bucket 'global-env-2'
'global-env-2/cgap-footest':
'{"fourfront": "http://portal-foo", "es": "http://es-foo", "ff_env": "cgap-footest"}',
'global-env-2/cgap-bartest':
'{"fourfront": "http://portal-bar", "es": "http://es-bar", "ff_env": "cgap-bartest"}',
}
with mock.patch.object(s3_utils_module, "boto3", MockBoto3(s3=MyS3)):
my_s3 = s3_utils_module.boto3.client('s3')
# Note here we specified the env explicitly.
config = EnvManager.verify_and_get_env_config(s3_client=my_s3, global_bucket='global-env-1', env='cgap-footest')
assert config['fourfront'] == 'http://portal'
assert config['es'] == 'http://es'
assert config['ff_env'] == 'cgap-footest'
env_manager_from_desc = EnvManager.compose(portal_url='http://portal',
es_url="http://es",
env_name='cgap-footest',
s3=my_s3)
assert env_manager_from_desc.env_description == config
assert env_manager_from_desc.env_description['fourfront'] == 'http://portal'
assert env_manager_from_desc.env_description['es'] == 'http://es'
assert env_manager_from_desc.env_description['ff_env'] == 'cgap-footest'
assert env_manager_from_desc.portal_url == 'http://portal'
assert env_manager_from_desc.es_url == 'http://es'
assert env_manager_from_desc.env_name == 'cgap-footest'
config = EnvManager.verify_and_get_env_config(s3_client=my_s3, global_bucket='global-env-1',
# Env unspecified, but there's only one, so it'll be inferred.
env=None)
assert config['fourfront'] == 'http://portal'
assert config['es'] == 'http://es'
assert config['ff_env'] == 'cgap-footest'
# The next tests are similar to the above, but in an S3 global bucket env (global-env-2) that has more than
# one environment, so the env cannot default.
config = EnvManager.verify_and_get_env_config(s3_client=my_s3, global_bucket='global-env-2', env='cgap-footest')
assert config['fourfront'] == 'http://portal-foo'
assert config['es'] == 'http://es-foo'
assert config['ff_env'] == 'cgap-footest'
config = EnvManager.verify_and_get_env_config(s3_client=my_s3, global_bucket='global-env-2', env='cgap-bartest')
assert config['fourfront'] == 'http://portal-bar'
assert config['es'] == 'http://es-bar'
assert config['ff_env'] == 'cgap-bartest'
with pytest.raises(CannotInferEnvFromManyGlobalEnvs):
EnvManager.verify_and_get_env_config(s3_client=my_s3, global_bucket='global-env-2',
# Env unspecified, but alas ambiguous, so no defaulting can occur.
env=None)