def __init__(self, parameter_store, stage_parameters, comparison_parameters): self.parameter_store = parameter_store self.stage_parameters = stage_parameters self.comparison_parameters = comparison_parameters self.s3 = S3(DEFAULT_REGION, S3_BUCKET_NAME) self.sts = STS()
def lambda_handler(event, _): # Return if we need to update the pipelines only if event.get('update_only'): LOGGER.info('Will only update pipelines for this execution.') return event try: sts = STS(boto3) parameter_store = ParameterStore( event.get('deployment_account_region'), boto3) for region in list( set([event.get('deployment_account_region')] + event.get("regions"))): kms_key_arn = parameter_store.fetch_parameter( "/cross_region/kms_arn/{0}".format(region)) s3_bucket = parameter_store.fetch_parameter( "/cross_region/s3_regional_bucket/{0}".format(region)) for account_id in event.get('account_ids'): try: role = sts.assume_cross_account_role( 'arn:aws:iam::{0}:role/{1}'.format( account_id, 'adf-cloudformation-deployment-role'), 'base_cfn_role') IAMUpdater(kms_key_arn, s3_bucket, role) kms = KMS(region, boto3, kms_key_arn, account_id) kms.enable_cross_account_access() except ClientError: continue return event except BaseException as error: LOGGER.exception(error)
def main(): parameter_store = ParameterStore(DEPLOYMENT_ACCOUNT_REGION, boto3) deployment_map = DeploymentMap(parameter_store, os.environ["ADF_PIPELINE_PREFIX"]) s3 = S3(DEPLOYMENT_ACCOUNT_REGION, boto3, S3_BUCKET_NAME) sts = STS(boto3) role = sts.assume_cross_account_role( 'arn:aws:iam::{0}:role/{1}-org-access-adf'.format( MASTER_ACCOUNT_ID, parameter_store.fetch_parameter('cross_account_access_role')), 'pipeline') organizations = Organizations(role) clean(parameter_store, deployment_map) for p in deployment_map.map_contents.get('pipelines'): pipeline = Pipeline(p) for target in p['targets']: target_structure = TargetStructure(target) for step in target_structure.target: for path in step.get('path'): try: regions = step.get( 'regions', p.get('regions', DEPLOYMENT_ACCOUNT_REGION)) pipeline.stage_regions.append(regions) pipeline_target = Target(path, regions, target_structure, organizations) pipeline_target.fetch_accounts_for_target() except BaseException: raise Exception( "Failed to return accounts for {0}".format(path)) pipeline.template_dictionary["targets"].append( target_structure.account_list) if DEPLOYMENT_ACCOUNT_REGION not in regions: pipeline.stage_regions.append(DEPLOYMENT_ACCOUNT_REGION) parameters = pipeline.generate_parameters() pipeline.generate() deployment_map.update_deployment_parameters(pipeline) s3_object_path = upload_if_required(s3, pipeline) store_regional_parameter_config(pipeline, parameter_store) cloudformation = CloudFormation( region=DEPLOYMENT_ACCOUNT_REGION, deployment_account_region=DEPLOYMENT_ACCOUNT_REGION, role=boto3, template_url=s3_object_path, parameters=parameters, wait=True, stack_name="{0}-{1}".format(os.environ["ADF_PIPELINE_PREFIX"], pipeline.name), s3=None, s3_key_path=None) cloudformation.create_stack()
def main(): LOGGER.info('ADF Version %s', ADF_VERSION) LOGGER.info("ADF Log Level is %s", ADF_LOG_LEVEL) parameter_store = ParameterStore(DEPLOYMENT_ACCOUNT_REGION, boto3) deployment_map = DeploymentMap(parameter_store, ADF_PIPELINE_PREFIX) s3 = S3(DEPLOYMENT_ACCOUNT_REGION, S3_BUCKET_NAME) sts = STS() role = sts.assume_cross_account_role( 'arn:aws:iam::{0}:role/{1}-readonly'.format( MASTER_ACCOUNT_ID, parameter_store.fetch_parameter('cross_account_access_role')), 'pipeline') organizations = Organizations(role) clean(parameter_store, deployment_map) try: auto_create_repositories = parameter_store.fetch_parameter( 'auto_create_repositories') except ParameterNotFoundError: auto_create_repositories = 'enabled' threads = [] for p in deployment_map.map_contents.get('pipelines'): thread = PropagatingThread(target=worker_thread, args=(p, organizations, auto_create_repositories, s3, deployment_map, parameter_store)) thread.start() threads.append(thread) for thread in threads: thread.join()
def __init__(self, parameter_store, stage_parameters, comparison_parameters): self.parameter_store = parameter_store self.stage_parameters = stage_parameters self.comparison_parameters = comparison_parameters self.sts = STS() self.cache = Cache()
def lambda_handler(event, _): sts = STS() role = sts.assume_cross_account_role( 'arn:aws:iam::{0}:role/{1}'.format( event["account_id"], event["cross_account_access_role"] ), 'master_lambda' ) if event['is_deployment_account']: configure_master_account_parameters(event) configure_deployment_account_parameters(event, role) s3 = S3( region=REGION_DEFAULT, bucket=S3_BUCKET ) for region in list(set([event["deployment_account_region"]] + event["regions"])): if not event["is_deployment_account"]: configure_generic_account(sts, event, region, role) cloudformation = CloudFormation( region=region, deployment_account_region=event["deployment_account_region"], role=role, wait=False, stack_name=None, # Stack name will be automatically defined based on event s3=s3, s3_key_path=event["full_path"], account_id=event["account_id"] ) cloudformation.create_stack() return event
def main(): #pylint: disable=R0915 LOGGER.info('ADF Version %s', ADF_VERSION) LOGGER.info("ADF Log Level is %s", ADF_LOG_LEVEL) parameter_store = ParameterStore(DEPLOYMENT_ACCOUNT_REGION, boto3) deployment_map = DeploymentMap(parameter_store, ADF_PIPELINE_PREFIX) s3 = S3(DEPLOYMENT_ACCOUNT_REGION, S3_BUCKET_NAME) sts = STS() role = sts.assume_cross_account_role( 'arn:aws:iam::{0}:role/{1}-readonly'.format( MASTER_ACCOUNT_ID, parameter_store.fetch_parameter('cross_account_access_role')), 'pipeline') organizations = Organizations(role) clean(parameter_store, deployment_map) for p in deployment_map.map_contents.get('pipelines'): pipeline = Pipeline(p) for target in p.get('targets', []): target_structure = TargetStructure(target) for step in target_structure.target: for path in step.get('path'): regions = step.get( 'regions', p.get('regions', DEPLOYMENT_ACCOUNT_REGION)) step_name = step.get('name') params = step.get('params', {}) pipeline.stage_regions.append(regions) pipeline_target = Target(path, regions, target_structure, organizations, step_name, params) pipeline_target.fetch_accounts_for_target() pipeline.template_dictionary["targets"].append( target_structure.account_list) if DEPLOYMENT_ACCOUNT_REGION not in regions: pipeline.stage_regions.append(DEPLOYMENT_ACCOUNT_REGION) parameters = pipeline.generate_parameters() pipeline.generate() deployment_map.update_deployment_parameters(pipeline) s3_object_path = upload_pipeline(s3, pipeline) store_regional_parameter_config(pipeline, parameter_store) cloudformation = CloudFormation( region=DEPLOYMENT_ACCOUNT_REGION, deployment_account_region=DEPLOYMENT_ACCOUNT_REGION, role=boto3, template_url=s3_object_path, parameters=parameters, wait=True, stack_name="{0}-{1}".format(ADF_PIPELINE_PREFIX, pipeline.name), s3=None, s3_key_path=None, account_id=DEPLOYMENT_ACCOUNT_ID) cloudformation.create_stack()
def create_or_update_account(org_session, support_session, account, adf_role_name, account_id=None): """Creates or updates a single AWS account. :param org_session: Instance of Organization class :param account: Instance of Account class """ if not account_id: LOGGER.info(f'Creating new account {account.full_name}') account_id = org_session.create_account(account, adf_role_name) # This only runs on account creation at the moment. support_session.set_support_level_for_account(account, account_id) sts = STS() role = sts.assume_cross_account_role( 'arn:aws:iam::{0}:role/{1}'.format( account_id, adf_role_name ), 'adf_account_provisioning' ) LOGGER.info(f'Ensuring account {account_id} (alias {account.alias}) is in OU {account.ou_path}') org_session.move_account(account_id, account.ou_path) if account.delete_default_vpc: ec2_client = role.client('ec2') all_regions = [ region['RegionName'] for region in ec2_client.describe_regions( AllRegions=False, Filters=[ { 'Name': 'opt-in-status', 'Values': [ 'opt-in-not-required', ] } ] )['Regions'] ] args = ( (account_id, region, role) for region in all_regions ) with ThreadPoolExecutor(max_workers=10) as executor: for _ in executor.map(lambda f: schedule_delete_default_vpc(*f), args): pass if account.alias: LOGGER.info(f'Ensuring account alias for {account_id} of {account.alias}') org_session.create_account_alias(account.alias, role) if account.tags: LOGGER.info(f'Ensuring tags exist for account {account_id}: {account.tags}') org_session.create_account_tags(account_id, account.tags)
def main(): LOGGER.info('ADF Version %s', ADF_VERSION) LOGGER.info("ADF Log Level is %s", ADF_LOG_LEVEL) parameter_store = ParameterStore( DEPLOYMENT_ACCOUNT_REGION, boto3 ) deployment_map = DeploymentMap( parameter_store, ADF_PIPELINE_PREFIX ) s3 = S3( DEPLOYMENT_ACCOUNT_REGION, S3_BUCKET_NAME ) sts = STS() role = sts.assume_cross_account_role( 'arn:aws:iam::{0}:role/{1}-readonly'.format( MASTER_ACCOUNT_ID, parameter_store.fetch_parameter('cross_account_access_role') ), 'pipeline' ) organizations = Organizations(role) clean(parameter_store, deployment_map) try: auto_create_repositories = parameter_store.fetch_parameter('auto_create_repositories') except ParameterNotFoundError: auto_create_repositories = 'enabled' threads = [] for counter, p in enumerate(deployment_map.map_contents.get('pipelines')): thread = PropagatingThread(target=worker_thread, args=( p, organizations, auto_create_repositories, s3, deployment_map, parameter_store )) thread.start() threads.append(thread) _batcher = counter % 10 if _batcher == 9: # 9 meaning we have hit a set of 10 threads since n % 10 _interval = random.randint(5, 11) LOGGER.debug('Waiting for %s seconds before starting next batch of 10 threads.', _interval) time.sleep(_interval) for thread in threads: thread.join()
def lambda_handler(event, _): if event.get("alias"): sts = STS() account_id = event.get("account_id") role = sts.assume_cross_account_role( f"arn:aws:iam::{account_id}:role/{ADF_ROLE_NAME}", "adf_account_alias_config", ) create_account_alias(event, role.client("iam")) else: LOGGER.info( f"Account: {event.get('account_full_name')} does not need an alias" ) return event
def lambda_handler(event, _): parameters = ParameterStore(REGION_DEFAULT, boto3) account_id = event.get( 'detail').get( 'requestParameters').get('accountId') organizations = Organizations(boto3, account_id) parsed_event = Event(event, parameters, organizations, account_id) cache = Cache() if parsed_event.moved_to_root or parsed_event.moved_to_protected: return parsed_event.create_output_object(cache) parsed_event.set_destination_ou_name() sts = STS(boto3) role = sts.assume_cross_account_role( 'arn:aws:iam::{0}:role/{1}'.format( parsed_event.account_id, parsed_event.cross_account_access_role ), 'master_lambda' ) if parsed_event.is_deployment_account: update_master_account_parameters(parsed_event, parameters) configure_deployment_account(parsed_event, role) s3 = S3(REGION_DEFAULT, boto3, S3_BUCKET) account_path = parsed_event.organizations.build_account_path( parsed_event.destination_ou_id, [], # Initial empty array to hold OU Path, cache, ) for region in list(set([parsed_event.deployment_account_region] + parsed_event.regions)): if not parsed_event.is_deployment_account: configure_generic_account(sts, parsed_event, region, role) cloudformation = CloudFormation( region=region, deployment_account_region=parsed_event.deployment_account_region, role=role, wait=False, stack_name=None, s3=s3, s3_key_path=account_path, file_path=None, ) cloudformation.create_stack() return parsed_event.create_output_object(cache)
def main(): LOGGER.info('ADF Version %s', ADF_VERSION) LOGGER.info("ADF Log Level is %s", ADF_LOG_LEVEL) _create_inputs_folder() parameter_store = ParameterStore( DEPLOYMENT_ACCOUNT_REGION, boto3 ) s3 = S3(DEPLOYMENT_ACCOUNT_REGION, SHARED_MODULES_BUCKET) deployment_map = DeploymentMap( parameter_store, s3, ADF_PIPELINE_PREFIX ) sts = STS() role = sts.assume_cross_account_role( 'arn:aws:iam::{0}:role/{1}-readonly'.format( MASTER_ACCOUNT_ID, parameter_store.fetch_parameter('cross_account_access_role') ), 'pipeline' ) organizations = Organizations(role) clean(parameter_store, deployment_map) ensure_event_bus_status(ORGANIZATION_ID) try: auto_create_repositories = parameter_store.fetch_parameter('auto_create_repositories') except ParameterNotFoundError: auto_create_repositories = 'enabled' threads = [] _cache = Cache() for p in deployment_map.map_contents.get('pipelines', []): _source_account_id = p.get('default_providers', {}).get('source', {}).get('properties', {}).get('account_id', {}) if _source_account_id and int(_source_account_id) != int(DEPLOYMENT_ACCOUNT_ID) and not _cache.check(_source_account_id): rule = Rule(p['default_providers']['source']['properties']['account_id']) rule.create_update() _cache.add(p['default_providers']['source']['properties']['account_id'], True) thread = PropagatingThread(target=worker_thread, args=( p, organizations, auto_create_repositories, deployment_map, parameter_store )) thread.start() threads.append(thread) for thread in threads: thread.join()
def lambda_handler(event, _): """Main Lambda Entry point """ sts = STS() account_id = event.get('account_id') partition = get_partition(REGION_DEFAULT) cross_account_access_role = event.get('cross_account_access_role') role = sts.assume_cross_account_role( f'arn:{partition}:iam::{account_id}:role/{cross_account_access_role}', 'master') s3 = S3(REGION_DEFAULT, S3_BUCKET) for region in list( set([event['deployment_account_region']] + event['regions'])): cloudformation = CloudFormation( region=region, deployment_account_region=event['deployment_account_region'], role=role, wait=False, stack_name=None, s3=s3, s3_key_path=event['ou_name'], account_id=account_id) status = cloudformation.get_stack_status() if status in ('CREATE_IN_PROGRESS', 'UPDATE_IN_PROGRESS'): raise RetryError(f"CloudFormation Stack status: {status}") if status in ('CREATE_FAILED', 'ROLLBACK_FAILED', 'DELETE_FAILED', 'UPDATE_ROLLBACK_FAILED', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE'): raise Exception( f"Account Bootstrap Failed - Account: {account_id} " f"Region: {region} Status: {status}") if event.get('is_deployment_account'): update_deployment_account_output_parameters( deployment_account_region=event['deployment_account_region'], region=region, deployment_account_role=role, cloudformation=cloudformation) return event
def lambda_handler(event, _): target_role_policies = { 'adf-cloudformation-deployment-role': 'adf-cloudformation-deployment-role-policy-kms', 'adf-cloudformation-role': 'adf-cloudformation-role-policy' } role_policies = { 'adf-codepipeline-role': 'adf-codepipeline-role-policy', 'adf-cloudformation-deployment-role': 'adf-cloudformation-deployment-role-policy', 'adf-cloudformation-role': 'adf-cloudformation-role-policy' } sts = STS() partition = get_partition(REGION_DEFAULT) parameter_store = ParameterStore( region=event.get('deployment_account_region'), role=boto3 ) account_id = event.get("account_id") kms_key_arns = [] s3_buckets = [] for region in list(set([event.get('deployment_account_region')] + event.get("regions", []))): kms_key_arn = parameter_store.fetch_parameter( f"/cross_region/kms_arn/{region}" ) kms_key_arns.append(kms_key_arn) s3_bucket = parameter_store.fetch_parameter( f"/cross_region/s3_regional_bucket/{region}" ) s3_buckets.append(s3_bucket) try: role = sts.assume_cross_account_role( f'arn:{partition}:iam::{account_id}:role/adf-cloudformation-deployment-role', 'base_cfn_role' ) LOGGER.debug("Role has been assumed for %s", account_id) update_iam(role, s3_bucket, kms_key_arn, target_role_policies) except ClientError as err: LOGGER.debug("%s could not be assumed (%s), continuing", account_id, err, exc_info=True) continue update_iam(boto3, s3_buckets, kms_key_arns, role_policies) return event
def lambda_handler(event, _): sts = STS(boto3) role = sts.assume_cross_account_role( 'arn:aws:iam::{0}:role/{1}'.format( event.get('deployment_account_id'), event.get('cross_account_iam_role')), 'step_function') step_functions = StepFunctions( role=role, deployment_account_id=event.get('deployment_account_id'), deployment_account_region=event.get('deployment_account_region'), regions=event.get('regions'), account_ids=[event.get('account_id')], update_pipelines_only=bool( event.get('moved_to_protected') or event.get('moved_to_root'))) step_functions.execute_statemachine() return event
def lambda_handler(event, _): """Main Lambda Entry point """ sts = STS(boto3) role = sts.assume_cross_account_role( 'arn:aws:iam::{0}:role/{1}'.format( event.get('account_id'), event.get('cross_account_iam_role'), ), 'master') s3 = S3(REGION_DEFAULT, boto3, S3_BUCKET) for region in list( set([event.get('deployment_account_region')] + event.get("regions"))): cloudformation = CloudFormation( region=region, deployment_account_region=event.get('deployment_account_region'), role=role, wait=False, stack_name=None, s3=s3, s3_key_path=event.get('ou_name'), file_path=None, ) status = cloudformation.get_stack_status() if status in ("CREATE_IN_PROGRESS", "UPDATE_IN_PROGRESS"): raise RetryError("Cloudformation Stack not yet complete") # TODO Better waiting validation to ensure stack is not failed if event.get('is_deployment_account'): update_deployment_account_output_parameters( deployment_account_region=event.get( 'deployment_account_region'), region=region, deployment_account_role=role, cloudformation=cloudformation) return event
def lambda_handler(event, _): target_role_policies = { 'adf-cloudformation-deployment-role': 'adf-cloudformation-deployment-role-policy-kms', 'adf-cloudformation-role': 'adf-cloudformation-role-policy' } role_policies = { 'adf-codepipeline-role': 'adf-codepipeline-role-policy', 'adf-cloudformation-deployment-role': 'adf-cloudformation-deployment-role-policy', 'adf-cloudformation-role': 'adf-cloudformation-role-policy' } sts = STS() parameter_store = ParameterStore( region=event.get('deployment_account_region'), role=boto3) for region in list( set([event.get('deployment_account_region')] + event.get("regions", []))): kms_key_arn = parameter_store.fetch_parameter( "/cross_region/kms_arn/{0}".format(region)) s3_bucket = parameter_store.fetch_parameter( "/cross_region/s3_regional_bucket/{0}".format(region)) update_iam(boto3, s3_bucket, kms_key_arn, role_policies) for account_id in event.get('account_ids'): try: role = sts.assume_cross_account_role( 'arn:aws:iam::{0}:role/{1}'.format( account_id, 'adf-cloudformation-deployment-role'), 'base_cfn_role') LOGGER.debug("Role has been assumed for %s", account_id) update_iam(role, s3_bucket, kms_key_arn, target_role_policies) except ClientError as err: LOGGER.debug("%s could not be assumed (%s), continuing", account_id, err, exc_info=True) continue return event
def lambda_handler(event, _): sts = STS() role = sts.assume_cross_account_role( 'arn:aws:iam::{0}:role/{1}'.format(event['deployment_account_id'], event['cross_account_access_role']), 'step_function') step_functions = StepFunctions( role=role, deployment_account_id=event['deployment_account_id'], deployment_account_region=event['deployment_account_region'], full_path=event['full_path'], regions=event['regions'], account_ids=[event['account_id']], update_pipelines_only=1 if event.get('moved_to_protected') or event.get('moved_to_root') else 0, error=event.get('error', 0)) step_functions.execute_statemachine() return event
def remove_base(account_id, regions, role, event): sts = STS() threads = [] for region in list(set([event.get('deployment_account_region')] + regions)): t = PropagatingThread(target=worker_thread, args=(sts, region, account_id, role, event)) t.start() threads.append(t) for thread in threads: thread.join()
def lambda_handler(event, _): sts = STS() account_id = event["account_id"] cross_account_access_role = event["cross_account_access_role"] role_arn = f'arn:{PARTITION}:iam::{account_id}:role/{cross_account_access_role}' role = sts.assume_cross_account_role(role_arn=role_arn, role_session_name='management_lambda') if event['is_deployment_account']: configure_master_account_parameters(event) configure_deployment_account_parameters(event, role) s3 = S3(region=REGION_DEFAULT, bucket=S3_BUCKET) for region in list( set([event["deployment_account_region"]] + event["regions"])): if not event["is_deployment_account"]: configure_generic_account(sts, event, region, role) cloudformation = CloudFormation( region=region, deployment_account_region=event["deployment_account_region"], role=role, wait=True, # Stack name will be automatically defined based on event stack_name=None, s3=s3, s3_key_path=event["full_path"], account_id=account_id) if is_inter_ou_account_move(event): cloudformation.delete_all_base_stacks(True) # override Wait cloudformation.create_stack() if region == event["deployment_account_region"]: cloudformation.create_iam_stack() return event
def lambda_handler(event, _): sts = STS(boto3) parameter_store = ParameterStore(event.get('deployment_account_region'), boto3) for region in list( set([event.get('deployment_account_region')] + event.get("regions"))): kms_key_arn = parameter_store.fetch_parameter( "/cross_region/kms_arn/{0}".format(region)) s3_bucket = parameter_store.fetch_parameter( "/cross_region/s3_regional_bucket/{0}".format(region)) for account_id in event.get('account_ids'): try: role = sts.assume_cross_account_role( 'arn:aws:iam::{0}:role/{1}'.format( account_id, 'adf-cloudformation-deployment-role'), 'base_cfn_role') IAMUpdater(kms_key_arn, s3_bucket, role) kms = KMS(region, boto3, kms_key_arn, account_id) kms.enable_cross_account_access() except ClientError: continue return event
def lambda_handler(event, _): LOGGER.info(f"Fetching Default regions {event.get('account_full_name')}") sts = STS() account_id = event.get("account_id") role = sts.assume_cross_account_role( f"arn:aws:iam::{account_id}:role/{ADF_ROLE_NAME}", "adf_account_get_regions", ) ec2_client = role.client("ec2") default_regions = [ region["RegionName"] for region in ec2_client.describe_regions( AllRegions=False, Filters=[{ "Name": "opt-in-status", "Values": ["opt-in-not-required", "opted-in"], }], )["Regions"] ] LOGGER.debug(f"Default regions for {account_id}: {default_regions}") return { **event, "default_regions": default_regions, }
def lambda_handler(event, _): sts = STS() deployment_account_id = event.get('deployment_account_id') partition = get_partition(REGION_DEFAULT) cross_account_access_role = event.get('cross_account_access_role') role = sts.assume_cross_account_role( f'arn:{partition}:iam::{deployment_account_id}:role/{cross_account_access_role}', 'step_function') step_functions = StepFunctions( role=role, deployment_account_id=deployment_account_id, deployment_account_region=event['deployment_account_region'], full_path=event['full_path'], regions=event['regions'], account_ids=[event['account_id']], update_pipelines_only=1 if event.get('moved_to_protected') or event.get('moved_to_root') else 0, error=event.get('error', 0)) step_functions.execute_statemachine() return event
def create_or_update_account(org_session, account, adf_role_name, account_id=None): """Creates or updates a single AWS account. :param org_session: Instance of Organization class :param account: Instance of Account class """ if not account_id: LOGGER.info(f'Creating new account {account.full_name}') account_id = org_session.create_account(account, adf_role_name) sts = STS() role = sts.assume_cross_account_role( 'arn:aws:iam::{0}:role/{1}'.format(account_id, adf_role_name), 'delete_default_vpc') LOGGER.info( f'Ensuring account {account_id} (alias {account.alias}) is in OU {account.ou_path}' ) org_session.move_account(account_id, account.ou_path) if account.delete_default_vpc: ec2_client = role.client('ec2') all_regions = get_all_regions(ec2_client) args = ((account_id, region, role) for region in all_regions) with ThreadPoolExecutor(max_workers=10) as executor: for _ in executor.map(lambda f: schedule_delete_default_vpc(*f), args): pass LOGGER.info(f'Ensuring account alias for {account_id} of {account.alias}') org_session.create_account_alias(account.alias, role) if account.tags: LOGGER.info( f'Ensuring tags exist for account {account_id}: {account.tags}') org_session.create_account_tags(account_id, account.tags)
def main(): config = Config() config.store_config() try: parameter_store = ParameterStore(REGION_DEFAULT, boto3) deployment_account_id = parameter_store.fetch_parameter( 'deployment_account_id') organizations = Organizations(boto3, deployment_account_id) sts = STS(boto3) deployment_account_role = prepare_deployment_account( sts=sts, deployment_account_id=deployment_account_id, config=config) cache = Cache() ou_id = organizations.get_parent_info().get("ou_parent_id") account_path = organizations.build_account_path(ou_id=ou_id, account_path=[], cache=cache) s3 = S3(REGION_DEFAULT, boto3, S3_BUCKET_NAME) # First Setup the Deployment Account in all regions (KMS Key and S3 Bucket + Parameter Store values) for region in list( set([config.deployment_account_region] + config.target_regions)): cloudformation = CloudFormation( region=region, deployment_account_region=config.deployment_account_region, role=deployment_account_role, wait=True, stack_name=None, s3=s3, s3_key_path=account_path) cloudformation.create_stack() update_deployment_account_output_parameters( deployment_account_region=config.deployment_account_region, region=region, deployment_account_role=deployment_account_role, cloudformation=cloudformation) threads = [] account_ids = organizations.get_account_ids() for account_id in account_ids: t = PropagatingThread(target=worker_thread, args=(account_id, sts, config, s3, cache)) t.start() threads.append(t) for thread in threads: thread.join() step_functions = StepFunctions( role=deployment_account_role, deployment_account_id=deployment_account_id, deployment_account_region=config.deployment_account_region, regions=config.target_regions, account_ids=[ i for i in account_ids if i != config.deployment_account_region ], update_pipelines_only=1) step_functions.execute_statemachine() except ParameterNotFoundError: LOGGER.info("Deployment Account has not yet been Bootstrapped.") return
def main(): # pylint: disable=R0915 LOGGER.info("ADF Version %s", ADF_VERSION) LOGGER.info("ADF Log Level is %s", ADF_LOG_LEVEL) policies = OrganizationPolicy() config = Config() config.store_config() try: parameter_store = ParameterStore(REGION_DEFAULT, boto3) deployment_account_id = parameter_store.fetch_parameter( 'deployment_account_id') organizations = Organizations(role=boto3, account_id=deployment_account_id) policies.apply(organizations, parameter_store, config.config) sts = STS() deployment_account_role = prepare_deployment_account( sts=sts, deployment_account_id=deployment_account_id, config=config) cache = Cache() ou_id = organizations.get_parent_info().get("ou_parent_id") account_path = organizations.build_account_path(ou_id=ou_id, account_path=[], cache=cache) s3 = S3(region=REGION_DEFAULT, bucket=S3_BUCKET_NAME) kms_and_bucket_dict = {} # First Setup/Update the Deployment Account in all regions (KMS Key and # S3 Bucket + Parameter Store values) for region in list( set([config.deployment_account_region] + config.target_regions)): cloudformation = CloudFormation( region=region, deployment_account_region=config.deployment_account_region, role=deployment_account_role, wait=True, stack_name=None, s3=s3, s3_key_path="adf-bootstrap/" + account_path, account_id=deployment_account_id) cloudformation.create_stack() update_deployment_account_output_parameters( deployment_account_region=config.deployment_account_region, region=region, kms_and_bucket_dict=kms_and_bucket_dict, deployment_account_role=deployment_account_role, cloudformation=cloudformation) if region == config.deployment_account_region: cloudformation.create_iam_stack() # Updating the stack on the master account in deployment region cloudformation = CloudFormation( region=config.deployment_account_region, deployment_account_region=config.deployment_account_region, role=boto3, wait=True, stack_name=None, s3=s3, s3_key_path='adf-build', account_id=ACCOUNT_ID) cloudformation.create_stack() threads = [] account_ids = [ account_id["Id"] for account_id in organizations.get_accounts() ] for account_id in [ account for account in account_ids if account != deployment_account_id ]: thread = PropagatingThread(target=worker_thread, args=(account_id, sts, config, s3, cache, kms_and_bucket_dict)) thread.start() threads.append(thread) for thread in threads: thread.join() LOGGER.info("Executing Step Function on Deployment Account") step_functions = StepFunctions( role=deployment_account_role, deployment_account_id=deployment_account_id, deployment_account_region=config.deployment_account_region, regions=config.target_regions, account_ids=account_ids, update_pipelines_only=0) step_functions.execute_statemachine() except ParameterNotFoundError: LOGGER.info( 'A Deployment Account is ready to be bootstrapped! ' 'The Account provisioner will now kick into action, ' 'be sure to check out its progress in AWS Step Functions in this account.' ) return
def main(): try: parameter_store = ParameterStore(REGION_DEFAULT, boto3) deployment_account_id = parameter_store.fetch_parameter( 'deployment_account_id') config = Config() config.store_config() if deployment_account_id is None: raise NotConfiguredError( "Deployment Account has not yet been configured") organizations = Organizations(boto3, deployment_account_id) sts = STS(boto3) ou_id = organizations.get_parent_info().get("ou_parent_id") deployment_account_role = ensure_deployment_account_configured( sts, config, ou_id, deployment_account_id) cache = Cache() account_path = organizations.build_account_path( ou_id, [], # Initial empty array to hold OU Path cache) s3 = S3(REGION_DEFAULT, boto3, S3_BUCKET) # (First) Setup the Deployment Account in all regions (KMS Key and S3 Bucket + Parameter Store values) for region in list( set([config.deployment_account_region] + config.target_regions)): cloudformation = CloudFormation( region=region, deployment_account_region=config.deployment_account_region, role=deployment_account_role, wait=True, stack_name=None, s3=s3, s3_key_path=account_path, file_path=None, ) cloudformation.create_stack() update_deployment_account_output_parameters( deployment_account_region=config.deployment_account_region, region=region, deployment_account_role=deployment_account_role, cloudformation=cloudformation) threads = [] account_ids = organizations.get_account_ids() for account_id in account_ids: t = PropagatingThread(target=worker_thread, args=(account_id, sts, config, s3, cache)) t.start() threads.append(t) for thread in threads: thread.join() step_functions = StepFunctions( role=deployment_account_role, deployment_account_id=deployment_account_id, deployment_account_region=config.deployment_account_region, regions=config.target_regions, account_ids=[ i for i in account_ids if i != config.deployment_account_region ], update_pipelines_only=False) step_functions.execute_statemachine() except NotConfiguredError as not_configured_error: LOGGER.info(not_configured_error)
properties associated with a pipeline. """ import os from cloudformation import CloudFormation from s3 import S3 from sts import STS from logger import configure_logger LOGGER = configure_logger(__name__) TARGET_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) DEPLOYMENT_ACCOUNT_ID = os.environ["ACCOUNT_ID"] DEPLOYMENT_ACCOUNT_REGION = os.environ.get("AWS_REGION", 'us-east-1') CODE_ACCOUNT_REGION = os.environ.get("AWS_REGION", 'us-east-1') S3_BUCKET_NAME = os.environ["S3_BUCKET_NAME"] sts = STS() s3 = S3(DEPLOYMENT_ACCOUNT_REGION, S3_BUCKET_NAME) class Repo: def __init__(self, account_id, name, description=''): self.name = name if not description: description = 'Created by ADF' self.description = description self.stack_name = "{0}-{1}".format('adf-codecommit', self.name) self.account_id = account_id self.session = sts.assume_cross_account_role( 'arn:aws:iam::{0}:role/adf-cloudformation-deployment-role'.format( account_id), 'create_repo_{0}'.format(account_id))
class Resolver: def __init__(self, parameter_store, stage_parameters, comparison_parameters): self.parameter_store = parameter_store self.stage_parameters = stage_parameters self.comparison_parameters = comparison_parameters self.sts = STS() self.cache = Cache() @staticmethod def _is_optional(value): return value.endswith('?') def fetch_stack_output(self, value, key, optional=False): # pylint: disable=too-many-statements try: [_, account_id, region, stack_name, output_key] = str(value).split(':') except ValueError: raise ValueError( "{0} is not a valid import string." "syntax should be import:account_id:region:stack_name:output_key" .format(str(value))) if Resolver._is_optional(output_key): LOGGER.info("Parameter %s is considered optional", output_key) optional = True output_key = output_key[:-1] if optional else output_key try: role = self.sts.assume_cross_account_role( 'arn:aws:iam::{0}:role/{1}'.format( account_id, 'adf-readonly-automation-role'), 'importer') cloudformation = CloudFormation( region=region, deployment_account_region=os.environ["AWS_REGION"], role=role, stack_name=stack_name, account_id=account_id) stack_output = self.cache.check( value) or cloudformation.get_stack_output(output_key) if stack_output: LOGGER.info("Stack output value is %s", stack_output) self.cache.add(value, stack_output) except ClientError: if not optional: raise stack_output = "" pass try: parent_key = list( Resolver.determine_parent_key(self.comparison_parameters, key))[0] if optional: self.stage_parameters[parent_key][key] = stack_output else: if not stack_output: raise Exception( "No Stack Output found on {account_id} in {region} " "with stack name {stack} and output key " "{output_key}".format( account_id=account_id, region=region, stack=stack_name, output_key=output_key, )) self.stage_parameters[parent_key][key] = stack_output except IndexError: if stack_output: if self.stage_parameters.get(key): self.stage_parameters[key] = stack_output else: raise Exception( "Could not determine the structure of the file in order to import from CloudFormation" ) return True def upload(self, value, key, file_name): if not any(item in value for item in ['path', 'virtual-hosted', 's3-key-only']): raise Exception( 'When uploading to S3 you need to specify a ' 'pathing style for the response either path or virtual-hosted, ' 'read more: https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html' ) from None if str(value).count(':') > 2: [_, region, style, value] = value.split(':') else: [_, style, value] = value.split(':') region = DEFAULT_REGION bucket_name = self.parameter_store.fetch_parameter( '/cross_region/s3_regional_bucket/{0}'.format(region)) client = S3(region, bucket_name) try: parent_key = list( Resolver.determine_parent_key(self.comparison_parameters, key))[0] except IndexError: if self.stage_parameters.get(key): self.stage_parameters[key] = client.put_object( "adf-upload/{0}/{1}".format(value, file_name), "{0}".format(value), style, True #pre-check ) return True self.stage_parameters[parent_key][key] = client.put_object( "adf-upload/{0}/{1}".format(value, file_name), "{0}".format(value), style, True #pre-check ) return True @staticmethod def determine_parent_key(d, target_key, parent_key=None): for key, value in d.items(): if key == target_key: yield parent_key if isinstance(value, dict): for result in Resolver.determine_parent_key( value, target_key, key): yield result def fetch_parameter_store_value(self, value, key, optional=False): # pylint: disable=too-many-statements if self._is_optional(value): LOGGER.info("Parameter %s is considered optional", value) optional = True if str(value).count(':') > 1: [_, region, value] = value.split(':') else: [_, value] = value.split(':') region = DEFAULT_REGION value = value[:-1] if optional else value client = ParameterStore(region, boto3) try: parameter = self.cache.check('{0}/{1}'.format( region, value)) or client.fetch_parameter(value) except ParameterNotFoundError: if optional: LOGGER.info("Parameter %s not found, returning empty string", value) parameter = "" else: raise try: parent_key = list( Resolver.determine_parent_key(self.comparison_parameters, key))[0] if parameter: self.cache.add('{0}/{1}'.format(region, value), parameter) self.stage_parameters[parent_key][key] = parameter except IndexError as error: if parameter: if self.stage_parameters.get(key): self.stage_parameters[key] = parameter else: LOGGER.error( "Parameter was not found, unable to fetch it from parameter store" ) raise Exception( "Parameter was not found, unable to fetch it from parameter store" ) from error return True def update(self, key): for k, _ in self.comparison_parameters.items(): if not self.stage_parameters.get( k) and not self.stage_parameters.get(k, {}).get(key): self.stage_parameters[k] = self.comparison_parameters[k] if key not in self.stage_parameters[ k] and self.comparison_parameters.get(k, {}).get(key): self.stage_parameters[k][key] = self.comparison_parameters[k][ key]
class Resolver: def __init__(self, parameter_store, stage_parameters, comparison_parameters): self.parameter_store = parameter_store self.stage_parameters = stage_parameters self.comparison_parameters = comparison_parameters self.s3 = S3(DEFAULT_REGION, S3_BUCKET_NAME) self.sts = STS() def fetch_stack_output(self, value, param, key=None): try: [_, account_id, region, stack_name, export] = str(value).split(':') except ValueError: raise ValueError( "{0} is not a valid import string." "syntax should be import:account_id:region:stack_name:export_key" .format(str(value))) LOGGER.info( "Assuming the role %s", 'arn:aws:iam::{0}:role/{1}'.format( account_id, 'adf-cloudformation-deployment-role')) role = self.sts.assume_cross_account_role( 'arn:aws:iam::{0}:role/{1}'.format( account_id, 'adf-cloudformation-deployment-role'), 'importer') cloudformation = CloudFormation( region=region, deployment_account_region=os.environ["AWS_REGION"], role=role, stack_name=stack_name, account_id=account_id) LOGGER.info("Retrieving value of key %s from %s on %s in %s", export, stack_name, account_id, region) stack_output = cloudformation.get_stack_output(export) if not stack_output: raise Exception("No Key was found on {0} with the name {1}".format( stack_name, export)) LOGGER.info("Stack output value is %s", stack_output) if key: self.stage_parameters[key][param] = stack_output return self.stage_parameters[key] = stack_output def upload(self, value, key, file_name, param=None): if str(value).count(':') > 1: [_, region, value] = value.split(':') bucket_name = self.parameter_store.fetch_parameter( '/cross_region/s3_regional_bucket/{0}'.format(region)) regional_client = S3(region, bucket_name) LOGGER.info("Uploading %s as %s to S3 Bucket %s in %s", value, file_name, bucket_name, region) if param: self.stage_parameters[param][key] = regional_client.put_object( "adf-upload/{0}/{1}".format(value, file_name), "{0}".format(value)) else: self.stage_parameters[key] = regional_client.put_object( "adf-upload/{0}/{1}".format(value, file_name), "{0}".format(value)) return True [_, value] = value.split(':') LOGGER.info("Uploading %s to S3", value) if param: self.stage_parameters[param][key] = self.s3.put_object( "adf-upload/{0}/{1}".format(value, file_name), "{0}".format(value)) else: self.stage_parameters[key] = self.s3.put_object( "adf-upload/{0}/{1}".format(value, file_name), "{0}".format(value)) return False def fetch_parameter_store_value(self, value, key, param=None): if str(value).count(':') > 1: [_, region, value] = value.split(':') regional_client = ParameterStore(region, boto3) LOGGER.info("Fetching Parameter from %s", value) if param: self.stage_parameters[param][ key] = regional_client.fetch_parameter(value) else: self.stage_parameters[key] = regional_client.fetch_parameter( value) return True [_, value] = value.split(':') LOGGER.info("Fetching Parameter from %s", value) if param: self.stage_parameters[param][ key] = self.parameter_store.fetch_parameter(value) else: self.stage_parameters[key] = self.parameter_store.fetch_parameter( value) return False def update_cfn(self, key, param): if key not in self.stage_parameters[param]: self.stage_parameters[param][key] = self.comparison_parameters[ param][key] def update_sc(self, key): if key not in self.stage_parameters: self.stage_parameters[key] = self.comparison_parameters[key]