def test_c4_orchestration_manager_find_stack(): mocked_boto3 = MockBoto3() mock_stack_names = [ 'foo-bar-baz', 'awseb-e-foo-stack', 'c4-network-cgap-test-123', 'c4-iam-cgap-test-456' ] MockBotoCloudFormationClient.setup_boto3_mocked_stacks( boto3=mocked_boto3, mock_stack_names=mock_stack_names) with mock.patch.object(cloudformation_utils, "boto3", mocked_boto3): manager = cloudformation_utils.C4OrchestrationManager() network_stack = manager.find_stack('network') assert network_stack.name == 'c4-network-cgap-test-123' mocked_boto3 = MockBoto3() mock_stack_names = [ 'foo-bar-baz', 'awseb-e-foo-stack', 'c4-network-cgap-test-123', 'c4-network-cgap-prod-456' ] MockBotoCloudFormationClient.setup_boto3_mocked_stacks( boto3=mocked_boto3, mock_stack_names=mock_stack_names) with mock.patch.object(cloudformation_utils, "boto3", mocked_boto3): manager = cloudformation_utils.C4OrchestrationManager() with pytest.raises(ValueError, match="too many"): manager.find_stack('network')
def test_get_ecs_real_url(): mocked_boto3 = MockBoto3() mastertest_url = 'http://C4EcsTrialAlphacgapmastertest-1234512345.us-east-1.elb.amazonaws.com' mastertest_outputs = [ { 'OutputKey': 'ECSApplicationURLcgapmastertest', 'OutputValue': mastertest_url, 'Description': 'URL of CGAP-Portal.', }, ] supertest_url = 'http://cgap-supertest-11223344.us-east-1.elb.amazonaws.com' supertest_outputs = [{ 'OutputKey': 'ECSApplicationURLcgapsupertest', 'OutputValue': supertest_url, 'Description': 'URL of CGAP-Portal.', }] mocked_stacks = [ MockBotoCloudFormationStack('c4-iam-main-stack'), MockBotoCloudFormationStack('c4-network-main-stack'), MockBotoCloudFormationStack('c4-ecs-cgap-supertest-stack', mock_outputs=supertest_outputs), MockBotoCloudFormationStack('c4-ecs-cgap-mastertest-stack', mock_outputs=mastertest_outputs), ] MockBotoCloudFormationClient.setup_boto3_mocked_stacks( boto3=mocked_boto3, mocked_stacks=mocked_stacks) with mock.patch.object(cloudformation_utils, "boto3", mocked_boto3): assert cloudformation_utils.get_ecs_real_url( 'cgap-supertest') == supertest_url assert cloudformation_utils.get_ecs_real_url( 'cgap-mastertest') == mastertest_url
def mocked_s3_integration(integrated_names=None, zip_suffix="", ffenv=None): """ This does common setup of some mocks needed by zip testing. """ zip_path_key = "zip_path" + zip_suffix zip_filename_key = "zip_filename" + zip_suffix b3 = MockBoto3() if not ffenv: ffenv = integrated_names['ffenv'] if integrated_names else None with mock.patch.object(s3_utils_module, "boto3", b3): s3_connection = s3Utils(env=ffenv) if integrated_names is not None: # Not needed when mocked. # s3_connection.s3_delete_dir(prefix) # In our mock, this won't exist already on S3 like in the integrated version of this test, # so we have to pre-load to our mock S3 manually. -kmp 13-Jan-2021 s3_connection.s3.upload_file(Filename=integrated_names[zip_path_key], Bucket=s3_connection.outfile_bucket, Key=integrated_names[zip_filename_key]) s3_connection.s3.put_object(Bucket=s3_connection.outfile_bucket, Key=integrated_names['filename'], Body=str.encode('thisisatest')) yield s3_connection
def boto3_for_some_secrets_testing(): mocked_boto3 = MockBoto3() manager = mocked_boto3.client('secretsmanager') manager.put_secret_value_for_testing(SecretId=some_secret_identity, Value=some_secret_string) manager.put_secret_value_for_testing(SecretId=decoy_1_identity, Value=decoy_1_string) manager.put_secret_value_for_testing(SecretId=decoy_2_identity, Value=decoy_2_string) return mocked_boto3
def test_c4_orchestration_manager_find_stack_resource(): mocked_boto3 = MockBoto3() private_subnet_a = MockBotoCloudFormationResourceSummary( logical_id='MyPrivateSubnetA', physical_resource_id='subnet-111111') private_subnet_b = MockBotoCloudFormationResourceSummary( logical_id='MyPrivateSubnetB', physical_resource_id='subnet-222222') public_subnet_a = MockBotoCloudFormationResourceSummary( logical_id='MyPublicSubnetA', physical_resource_id='subnet-333333') public_subnet_b = MockBotoCloudFormationResourceSummary( logical_id='MyPublicSubnetA', physical_resource_id='subnet-444444') network_resource_summaries = [ private_subnet_a, private_subnet_b, public_subnet_a, public_subnet_b ] mocked_stacks = [ MockBotoCloudFormationStack( 'c4-network-main-stack', mock_resource_summaries=network_resource_summaries), MockBotoCloudFormationStack('c4-iam-main-stack') ] MockBotoCloudFormationClient.setup_boto3_mocked_stacks( boto3=mocked_boto3, mocked_stacks=mocked_stacks) with mock.patch.object(cloudformation_utils, "boto3", mocked_boto3): manager = cloudformation_utils.C4OrchestrationManager() assert manager.find_stack_resource( 'network', 'MyPrivateSubnetA') == private_subnet_a assert manager.find_stack_resource( 'network', 'MyPrivateSubnetB') == private_subnet_b assert manager.find_stack_resource('network', 'MyPrivateSubnetC') is None assert manager.find_stack_resource('network', 'MyPrivateSubnetA', 'logical_id') == 'MyPrivateSubnetA' assert manager.find_stack_resource('network', 'MyPrivateSubnetC', 'logical_id') is None assert manager.find_stack_resource( 'network', 'MyPrivateSubnetA', 'physical_resource_id') == 'subnet-111111' assert manager.find_stack_resource('network', 'MyPrivateSubnetC', 'physical_resource_id') is None assert manager.find_stack_resource('network', 'MyPrivateSubnetC', 'physical_resource_id', 'foo') == 'foo'
def test_c4_orchestration_manager_find_stack_outputs(): mocked_boto3 = MockBoto3() network_outputs = [ { 'OutputKey': 'rds', 'OutputValue': 'some-db-thing' }, { 'OutputKey': 'es', 'OutputValue': 'some-es-thing' }, ] iam_outputs = [ { 'OutputKey': 'user1', 'OutputValue': 'Joe' }, { 'OutputKey': 'user2', 'OutputValue': 'Sally' }, ] mocked_stacks = [ MockBotoCloudFormationStack('c4-network-main-stack', mock_outputs=network_outputs), MockBotoCloudFormationStack('c4-iam-main-stack', mock_outputs=iam_outputs), ] MockBotoCloudFormationClient.setup_boto3_mocked_stacks( boto3=mocked_boto3, mocked_stacks=mocked_stacks) with mock.patch.object(cloudformation_utils, "boto3", mocked_boto3): manager = cloudformation_utils.C4OrchestrationManager() # Looking up a simple key by name assert manager.find_stack_outputs('user1') == {'user1': 'Joe'} assert manager.find_stack_outputs('user1', value_only=True) == ['Joe'] assert manager.find_stack_outputs('rds') == {'rds': 'some-db-thing'} assert manager.find_stack_outputs( 'rds', value_only=True) == ['some-db-thing'] assert manager.find_stack_outputs('not-there') == {} assert manager.find_stack_outputs('not-there', value_only=True) == [] # Use of predicate to find several related keys assert manager.find_stack_outputs(lambda x: x.startswith("user")) == { 'user1': 'Joe', 'user2': 'Sally' } assert sorted( manager.find_stack_outputs(lambda x: x.startswith("user"), value_only=True)) == ['Joe', 'Sally']
def test_get_ecr_repo_url_hybrid(use_ecosystem_repo): main_repo = 'xxx6742.dkr.ecr.us-east-1.amazonaws.com/main' main_stack = 'c4-ecr-main-stack' main_outputs = [ { 'OutputKey': 'C4ECRMainRepoURL', 'OutputValue': main_repo, 'Description': 'CGAPDocker Image Repository URL', 'ExportName': f'{main_stack}-RepoURL' }, ] mocked_boto3 = MockBoto3() mastertest_repo = 'xxx6742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest' mastertest_stack = 'c4-ecr-cgap-mastertest-stack' mastertest_outputs = [ { 'OutputKey': 'C4EcrCgapMastertestRepoURL', 'OutputValue': mastertest_repo, 'Description': 'CGAPDocker Image Repository URL', 'ExportName': f'{mastertest_stack}-RepoURL' }, ] hotseat_repo = 'xxx6742.dkr.ecr.us-east-1.amazonaws.com/cgap-hotseat' hotseat_stack = 'c4-ecr-cgap-hotseat-stack' hotseat_outputs = [ { 'OutputKey': 'C4EcrCgapHotseatRepoURL', 'OutputValue': hotseat_repo, 'Description': 'CGAPDocker Image Repository URL', 'ExportName': f'{hotseat_stack}-RepoURL' }, ] mocked_stacks = [ MockBotoCloudFormationStack('c4-iam-trial-alpha-stack'), MockBotoCloudFormationStack('c4-network-trial-alpha-stack'), MockBotoCloudFormationStack(mastertest_stack, mock_outputs=mastertest_outputs), MockBotoCloudFormationStack(hotseat_stack, mock_outputs=hotseat_outputs), ] if use_ecosystem_repo: mocked_stacks.append( MockBotoCloudFormationStack(main_stack, mock_outputs=main_outputs)) MockBotoCloudFormationClient.setup_boto3_mocked_stacks( boto3=mocked_boto3, mocked_stacks=mocked_stacks) with mock.patch.object(cloudformation_utils, "boto3", mocked_boto3): assert cloudformation_utils.get_ecr_repo_url( 'cgap-mastertest') == mastertest_repo assert cloudformation_utils.get_ecr_repo_url( env_name='cgap-hotseat') == hotseat_repo assert cloudformation_utils.get_ecr_repo_url( env_name='cgap-wolf') == (main_repo if use_ecosystem_repo else '')
def test_c4_orchestration_manager_all_stacks(): mocked_boto3 = MockBoto3() mock_stack_names = [ 'foo-bar-baz', 'awseb-e-123-stack', 'c4-foo-123', 'c4-foo-456' ] MockBotoCloudFormationClient.setup_boto3_mocked_stacks( boto3=mocked_boto3, mock_stack_names=mock_stack_names) with mock.patch.object(cloudformation_utils, "boto3", mocked_boto3): manager = cloudformation_utils.C4OrchestrationManager() all_stacks = manager.all_stacks() assert sorted(s.name for s in all_stacks) == ['c4-foo-123', 'c4-foo-456']
def mocked_s3utils(beanstalks=None, require_sse=False, other_access_key_names=None): """ This context manager sets up a mock version of boto3 for use by s3_utils and ff_utils during the context of its test. It also sets up the S3_ENCRYPT_KEY environment variable with a sample value for testing, and it sets up a set of mocked beanstalks for fourfront-foo and fourfront-bar, so that s3_utils will not get confused when it does discovery operations to find them. """ if beanstalks is None: beanstalks = TestScenarios.DEFAULT_BEANSTALKS # First we make a mocked boto3 that will use an S3 mock with mock server side encryption. s3_class = (make_mock_boto_s3_with_sse( beanstalks=beanstalks, other_access_key_names=other_access_key_names) if require_sse else MockBotoS3Client) mock_boto3 = MockBoto3( s3=s3_class, elasticbeanstalk=make_mock_boto_eb_client_class(beanstalks=beanstalks)) # Now we arrange that both s3_utils and ff_utils modules share the illusion that our mock IS the boto3 library with mock.patch.object(s3_utils, "boto3", mock_boto3): with mock.patch.object(ff_utils, "boto3", mock_boto3): with mock.patch.object(beanstalk_utils, "boto3", mock_boto3): with mock.patch.object(base, "boto3", mock_boto3): with mock.patch.object( s3_utils.EnvManager, "fetch_health_page_json") as mock_fetcher: # This is all that's needed for s3Utils to initialize an EnvManager. # We might have to add more later. def mocked_fetch_health_page_json( url, use_urllib=True): ignored(use_urllib) # we don't test this m = re.match(r'.*(fourfront-[a-z0-9-]+)(?:[.]|$)', url) if m: env_name = m.group(1) return make_mock_health_page(env_name) else: raise NotImplementedError( f"Mock can't handle URL: {url}") mock_fetcher.side_effect = mocked_fetch_health_page_json # The mocked encrypt key is expected by various tools in the s3_utils module to be supplied # as an environment variable (i.e., in os.environ), so this sets up that environment variable. if require_sse: with override_environ( S3_ENCRYPT_KEY=s3_class.SSE_ENCRYPT_KEY): yield else: yield
def test_c4_orchestration_manager_extract_stack_name_token(): mocked_boto3 = MockBoto3() with mock.patch.object(cloudformation_utils, "boto3", mocked_boto3): def extract(stack_name): stack = MockBotoCloudFormationStack(stack_name) manager = cloudformation_utils.C4OrchestrationManager() return manager._extract_stack_name_token( stack) # noQA - I know this is protected. Just testing it. assert extract('foo-bar') is None assert extract('c4-foo') == 'foo' assert extract('c4-foo-bar') == 'foo' assert extract('c4-foo-bar-baz') == 'foo' assert extract('c4-foo_bar-baz') == 'foo_bar'
def test_c4_orchestration_manager_get_stack_output(): mocked_boto3 = MockBoto3() with mock.patch.object(cloudformation_utils, "boto3", mocked_boto3): manager = cloudformation_utils.C4OrchestrationManager() mock_outputs = [{ 'OutputKey': 'alpha', 'OutputValue': 'one' }, { 'OutputKey': 'beta', 'OutputValue': 'two' }] s = MockBotoCloudFormationStack('foo', mock_outputs=mock_outputs) assert manager.get_stack_output(s, 'alpha') == 'one' assert manager.get_stack_output(s, 'beta') == 'two' assert manager.get_stack_output(s, 'gamma') is None
def test_awseb_orchestration_manager_extract_stack_name_token(): mocked_boto3 = MockBoto3() with mock.patch.object(cloudformation_utils, "boto3", mocked_boto3): def extract(stack_name): stack = MockBotoCloudFormationStack(stack_name) manager = cloudformation_utils.AwsebOrchestrationManager() return manager._extract_stack_name_token( stack ) # noQA - please don't tell me this is protected. Just testing assert extract('foo-bar') is None assert extract('c4-foo') is None assert extract('awseb-foo-bar') is None assert extract('awseb-x-bar') is None assert extract('awseb-e-bar') is None assert extract('awseb-e-bar-stack') == 'bar' assert extract('awseb-e-bar-baz-stack') == 'bar-baz'
def test_get_ecr_repo_url_kmp( ): # Test for my test environment. -kmp 17-Aug-2021 mocked_boto3 = MockBoto3() supertest_repo = 'xxx0312.dkr.ecr.us-east-1.amazonaws.com/main' supertest_outputs = [{ 'OutputKey': 'C4ECRMainRepoURL', 'OutputValue': supertest_repo, 'Description': 'CGAPDocker Image Repository URL', 'ExportName': 'c4-ecr-main-stack-RepoURL' }] mocked_stacks = [ MockBotoCloudFormationStack('c4-iam-trial-alpha-stack'), MockBotoCloudFormationStack('c4-network-trial-alpha-stack'), MockBotoCloudFormationStack('c4-ecr-trial-alpha-stack', mock_outputs=supertest_outputs), ] MockBotoCloudFormationClient.setup_boto3_mocked_stacks( boto3=mocked_boto3, mocked_stacks=mocked_stacks) with mock.patch.object(cloudformation_utils, "boto3", mocked_boto3): assert cloudformation_utils.get_ecr_repo_url( 'cgap-supertest') == supertest_repo
def test_get_ecr_repo_url_alpha(): mocked_boto3 = MockBoto3() mastertest_repo = 'xxx6742.dkr.ecr.us-east-1.amazonaws.com/cgap-mastertest' mastertest_outputs = [ { 'OutputKey': 'C4EcrTrialAlphaECRRepoURL', 'OutputValue': mastertest_repo, 'Description': 'CGAPDocker Image Repository URL', 'ExportName': 'c4-ecr-trial-alpha-stack-ECRRepoURL' }, ] mocked_stacks = [ MockBotoCloudFormationStack('c4-iam-trial-alpha-stack'), MockBotoCloudFormationStack('c4-network-trial-alpha-stack'), MockBotoCloudFormationStack('c4-ecr-trial-alpha-stack', mock_outputs=mastertest_outputs), ] MockBotoCloudFormationClient.setup_boto3_mocked_stacks( boto3=mocked_boto3, mocked_stacks=mocked_stacks) with mock.patch.object(cloudformation_utils, "boto3", mocked_boto3): assert cloudformation_utils.get_ecr_repo_url( 'cgap-mastertest') == mastertest_repo
def test_env_manager(): class MyS3(MockBotoS3Client): MOCK_STATIC_FILES = { # Bucket 'global-env-1' 'global-env-1/cgap-footest': '{"fourfront": "http://portal", "es": "http://es", "ff_env": "cgap-footest"}', # Bucket 'global-env-2' 'global-env-2/cgap-footest': '{"fourfront": "http://portal-foo", "es": "http://es-foo", "ff_env": "cgap-footest"}', 'global-env-2/cgap-bartest': '{"fourfront": "http://portal-bar", "es": "http://es-bar", "ff_env": "cgap-bartest"}', } with mock.patch.object(s3_utils_module, "boto3", MockBoto3(s3=MyS3)): my_s3 = s3_utils_module.boto3.client('s3') with EnvManager.global_env_bucket_named(name='global-env-1'): e = EnvManager(s3=my_s3) assert e.portal_url == "http://portal" assert e.es_url == "http://es" assert e.env_name == "cgap-footest"
def test_env_manager_verify_and_get_env_config(): class MyS3(MockBotoS3Client): MOCK_STATIC_FILES = { # Bucket 'global-env-1' 'global-env-1/cgap-footest': '{"fourfront": "http://portal", "es": "http://es", "ff_env": "cgap-footest"}', # Bucket 'global-env-2' 'global-env-2/cgap-footest': '{"fourfront": "http://portal-foo", "es": "http://es-foo", "ff_env": "cgap-footest"}', 'global-env-2/cgap-bartest': '{"fourfront": "http://portal-bar", "es": "http://es-bar", "ff_env": "cgap-bartest"}', } with mock.patch.object(s3_utils_module, "boto3", MockBoto3(s3=MyS3)): my_s3 = s3_utils_module.boto3.client('s3') # Note here we specified the env explicitly. config = EnvManager.verify_and_get_env_config(s3_client=my_s3, global_bucket='global-env-1', env='cgap-footest') assert config['fourfront'] == 'http://portal' assert config['es'] == 'http://es' assert config['ff_env'] == 'cgap-footest' env_manager_from_desc = EnvManager.compose(portal_url='http://portal', es_url="http://es", env_name='cgap-footest', s3=my_s3) assert env_manager_from_desc.env_description == config assert env_manager_from_desc.env_description['fourfront'] == 'http://portal' assert env_manager_from_desc.env_description['es'] == 'http://es' assert env_manager_from_desc.env_description['ff_env'] == 'cgap-footest' assert env_manager_from_desc.portal_url == 'http://portal' assert env_manager_from_desc.es_url == 'http://es' assert env_manager_from_desc.env_name == 'cgap-footest' config = EnvManager.verify_and_get_env_config(s3_client=my_s3, global_bucket='global-env-1', # Env unspecified, but there's only one, so it'll be inferred. env=None) assert config['fourfront'] == 'http://portal' assert config['es'] == 'http://es' assert config['ff_env'] == 'cgap-footest' # The next tests are similar to the above, but in an S3 global bucket env (global-env-2) that has more than # one environment, so the env cannot default. config = EnvManager.verify_and_get_env_config(s3_client=my_s3, global_bucket='global-env-2', env='cgap-footest') assert config['fourfront'] == 'http://portal-foo' assert config['es'] == 'http://es-foo' assert config['ff_env'] == 'cgap-footest' config = EnvManager.verify_and_get_env_config(s3_client=my_s3, global_bucket='global-env-2', env='cgap-bartest') assert config['fourfront'] == 'http://portal-bar' assert config['es'] == 'http://es-bar' assert config['ff_env'] == 'cgap-bartest' with pytest.raises(CannotInferEnvFromManyGlobalEnvs): EnvManager.verify_and_get_env_config(s3_client=my_s3, global_bucket='global-env-2', # Env unspecified, but alas ambiguous, so no defaulting can occur. env=None)