def remove_base(account_id, regions, role, event):
sts = STS()
threads = []
for region in list(set([event.get('deployment_account_region')] +
regions)):
t = PropagatingThread(target=worker_thread,
args=(sts, region, account_id, role, event))
t.start()
threads.append(t)
for thread in threads:
thread.join()
def main():
LOGGER.info('ADF Version %s', ADF_VERSION)
LOGGER.info("ADF Log Level is %s", ADF_LOG_LEVEL)
s3 = S3(
DEPLOYMENT_ACCOUNT_REGION,
S3_BUCKET_NAME
)
threads = []
_templates = glob.glob("cdk.out/*.template.json")
for counter, template_path in enumerate(_templates):
name = os.path.splitext(template_path.split('/')[-1].split('.template')[0])[0] # Just stackname no extension and no .template
with open(template_path) as _template_path:
thread = PropagatingThread(target=worker_thread, args=(
template_path,
name,
s3
))
thread.start()
threads.append(thread)
_batcher = counter % 10
if _batcher == 9: # 9 meaning we have hit a set of 10 threads since n % 10
_interval = random.randint(5, 11)
LOGGER.debug('Waiting for %s seconds before starting next batch of 10 threads.', _interval)
time.sleep(_interval)
for thread in threads:
thread.join()
def main():
LOGGER.info('ADF Version %s', ADF_VERSION)
LOGGER.info("ADF Log Level is %s", ADF_LOG_LEVEL)
parameter_store = ParameterStore(DEPLOYMENT_ACCOUNT_REGION, boto3)
deployment_map = DeploymentMap(parameter_store, ADF_PIPELINE_PREFIX)
s3 = S3(DEPLOYMENT_ACCOUNT_REGION, S3_BUCKET_NAME)
sts = STS()
role = sts.assume_cross_account_role(
'arn:aws:iam::{0}:role/{1}-readonly'.format(
MASTER_ACCOUNT_ID,
parameter_store.fetch_parameter('cross_account_access_role')),
'pipeline')
organizations = Organizations(role)
clean(parameter_store, deployment_map)
try:
auto_create_repositories = parameter_store.fetch_parameter(
'auto_create_repositories')
except ParameterNotFoundError:
auto_create_repositories = 'enabled'
threads = []
for p in deployment_map.map_contents.get('pipelines'):
thread = PropagatingThread(target=worker_thread,
args=(p, organizations,
auto_create_repositories, s3,
deployment_map, parameter_store))
thread.start()
threads.append(thread)
for thread in threads:
thread.join()
def main():
LOGGER.info('ADF Version %s', ADF_VERSION)
LOGGER.info("ADF Log Level is %s", ADF_LOG_LEVEL)
parameter_store = ParameterStore(
DEPLOYMENT_ACCOUNT_REGION,
boto3
)
deployment_map = DeploymentMap(
parameter_store,
ADF_PIPELINE_PREFIX
)
s3 = S3(
DEPLOYMENT_ACCOUNT_REGION,
S3_BUCKET_NAME
)
sts = STS()
role = sts.assume_cross_account_role(
'arn:aws:iam::{0}:role/{1}-readonly'.format(
MASTER_ACCOUNT_ID,
parameter_store.fetch_parameter('cross_account_access_role')
), 'pipeline'
)
organizations = Organizations(role)
clean(parameter_store, deployment_map)
try:
auto_create_repositories = parameter_store.fetch_parameter('auto_create_repositories')
except ParameterNotFoundError:
auto_create_repositories = 'enabled'
threads = []
for counter, p in enumerate(deployment_map.map_contents.get('pipelines')):
thread = PropagatingThread(target=worker_thread, args=(
p,
organizations,
auto_create_repositories,
s3,
deployment_map,
parameter_store
))
thread.start()
threads.append(thread)
_batcher = counter % 10
if _batcher == 9: # 9 meaning we have hit a set of 10 threads since n % 10
_interval = random.randint(5, 11)
LOGGER.debug('Waiting for %s seconds before starting next batch of 10 threads.', _interval)
time.sleep(_interval)
for thread in threads:
thread.join()
def main():
LOGGER.info('ADF Version %s', ADF_VERSION)
LOGGER.info("ADF Log Level is %s", ADF_LOG_LEVEL)
_create_inputs_folder()
parameter_store = ParameterStore(
DEPLOYMENT_ACCOUNT_REGION,
boto3
)
s3 = S3(DEPLOYMENT_ACCOUNT_REGION, SHARED_MODULES_BUCKET)
deployment_map = DeploymentMap(
parameter_store,
s3,
ADF_PIPELINE_PREFIX
)
sts = STS()
role = sts.assume_cross_account_role(
'arn:aws:iam::{0}:role/{1}-readonly'.format(
MASTER_ACCOUNT_ID,
parameter_store.fetch_parameter('cross_account_access_role')
), 'pipeline'
)
organizations = Organizations(role)
clean(parameter_store, deployment_map)
ensure_event_bus_status(ORGANIZATION_ID)
try:
auto_create_repositories = parameter_store.fetch_parameter('auto_create_repositories')
except ParameterNotFoundError:
auto_create_repositories = 'enabled'
threads = []
_cache = Cache()
for p in deployment_map.map_contents.get('pipelines', []):
_source_account_id = p.get('default_providers', {}).get('source', {}).get('properties', {}).get('account_id', {})
if _source_account_id and int(_source_account_id) != int(DEPLOYMENT_ACCOUNT_ID) and not _cache.check(_source_account_id):
rule = Rule(p['default_providers']['source']['properties']['account_id'])
rule.create_update()
_cache.add(p['default_providers']['source']['properties']['account_id'], True)
thread = PropagatingThread(target=worker_thread, args=(
p,
organizations,
auto_create_repositories,
deployment_map,
parameter_store
))
thread.start()
threads.append(thread)
for thread in threads:
thread.join()
def main():
config = Config()
config.store_config()
try:
parameter_store = ParameterStore(REGION_DEFAULT, boto3)
deployment_account_id = parameter_store.fetch_parameter(
'deployment_account_id')
organizations = Organizations(boto3, deployment_account_id)
sts = STS(boto3)
deployment_account_role = prepare_deployment_account(
sts=sts,
deployment_account_id=deployment_account_id,
config=config)
cache = Cache()
ou_id = organizations.get_parent_info().get("ou_parent_id")
account_path = organizations.build_account_path(ou_id=ou_id,
account_path=[],
cache=cache)
s3 = S3(REGION_DEFAULT, boto3, S3_BUCKET_NAME)
# First Setup the Deployment Account in all regions (KMS Key and S3 Bucket + Parameter Store values)
for region in list(
set([config.deployment_account_region] +
config.target_regions)):
cloudformation = CloudFormation(
region=region,
deployment_account_region=config.deployment_account_region,
role=deployment_account_role,
wait=True,
stack_name=None,
s3=s3,
s3_key_path=account_path)
cloudformation.create_stack()
update_deployment_account_output_parameters(
deployment_account_region=config.deployment_account_region,
region=region,
deployment_account_role=deployment_account_role,
cloudformation=cloudformation)
threads = []
account_ids = organizations.get_account_ids()
for account_id in account_ids:
t = PropagatingThread(target=worker_thread,
args=(account_id, sts, config, s3, cache))
t.start()
threads.append(t)
for thread in threads:
thread.join()
step_functions = StepFunctions(
role=deployment_account_role,
deployment_account_id=deployment_account_id,
deployment_account_region=config.deployment_account_region,
regions=config.target_regions,
account_ids=[
i for i in account_ids if i != config.deployment_account_region
],
update_pipelines_only=1)
step_functions.execute_statemachine()
except ParameterNotFoundError:
LOGGER.info("Deployment Account has not yet been Bootstrapped.")
return
def main(): # pylint: disable=R0915
LOGGER.info("ADF Version %s", ADF_VERSION)
LOGGER.info("ADF Log Level is %s", ADF_LOG_LEVEL)
policies = OrganizationPolicy()
config = Config()
config.store_config()
try:
parameter_store = ParameterStore(REGION_DEFAULT, boto3)
deployment_account_id = parameter_store.fetch_parameter(
'deployment_account_id')
organizations = Organizations(role=boto3,
account_id=deployment_account_id)
policies.apply(organizations, parameter_store, config.config)
sts = STS()
deployment_account_role = prepare_deployment_account(
sts=sts,
deployment_account_id=deployment_account_id,
config=config)
cache = Cache()
ou_id = organizations.get_parent_info().get("ou_parent_id")
account_path = organizations.build_account_path(ou_id=ou_id,
account_path=[],
cache=cache)
s3 = S3(region=REGION_DEFAULT, bucket=S3_BUCKET_NAME)
kms_and_bucket_dict = {}
# First Setup/Update the Deployment Account in all regions (KMS Key and
# S3 Bucket + Parameter Store values)
for region in list(
set([config.deployment_account_region] +
config.target_regions)):
cloudformation = CloudFormation(
region=region,
deployment_account_region=config.deployment_account_region,
role=deployment_account_role,
wait=True,
stack_name=None,
s3=s3,
s3_key_path="adf-bootstrap/" + account_path,
account_id=deployment_account_id)
cloudformation.create_stack()
update_deployment_account_output_parameters(
deployment_account_region=config.deployment_account_region,
region=region,
kms_and_bucket_dict=kms_and_bucket_dict,
deployment_account_role=deployment_account_role,
cloudformation=cloudformation)
if region == config.deployment_account_region:
cloudformation.create_iam_stack()
# Updating the stack on the master account in deployment region
cloudformation = CloudFormation(
region=config.deployment_account_region,
deployment_account_region=config.deployment_account_region,
role=boto3,
wait=True,
stack_name=None,
s3=s3,
s3_key_path='adf-build',
account_id=ACCOUNT_ID)
cloudformation.create_stack()
threads = []
account_ids = [
account_id["Id"] for account_id in organizations.get_accounts()
]
for account_id in [
account for account in account_ids
if account != deployment_account_id
]:
thread = PropagatingThread(target=worker_thread,
args=(account_id, sts, config, s3,
cache, kms_and_bucket_dict))
thread.start()
threads.append(thread)
for thread in threads:
thread.join()
LOGGER.info("Executing Step Function on Deployment Account")
step_functions = StepFunctions(
role=deployment_account_role,
deployment_account_id=deployment_account_id,
deployment_account_region=config.deployment_account_region,
regions=config.target_regions,
account_ids=account_ids,
update_pipelines_only=0)
step_functions.execute_statemachine()
except ParameterNotFoundError:
LOGGER.info(
'A Deployment Account is ready to be bootstrapped! '
'The Account provisioner will now kick into action, '
'be sure to check out its progress in AWS Step Functions in this account.'
)
return
def main():
try:
parameter_store = ParameterStore(REGION_DEFAULT, boto3)
deployment_account_id = parameter_store.fetch_parameter(
'deployment_account_id')
config = Config()
config.store_config()
if deployment_account_id is None:
raise NotConfiguredError(
"Deployment Account has not yet been configured")
organizations = Organizations(boto3, deployment_account_id)
sts = STS(boto3)
ou_id = organizations.get_parent_info().get("ou_parent_id")
deployment_account_role = ensure_deployment_account_configured(
sts, config, ou_id, deployment_account_id)
cache = Cache()
account_path = organizations.build_account_path(
ou_id,
[], # Initial empty array to hold OU Path
cache)
s3 = S3(REGION_DEFAULT, boto3, S3_BUCKET)
# (First) Setup the Deployment Account in all regions (KMS Key and S3 Bucket + Parameter Store values)
for region in list(
set([config.deployment_account_region] +
config.target_regions)):
cloudformation = CloudFormation(
region=region,
deployment_account_region=config.deployment_account_region,
role=deployment_account_role,
wait=True,
stack_name=None,
s3=s3,
s3_key_path=account_path,
file_path=None,
)
cloudformation.create_stack()
update_deployment_account_output_parameters(
deployment_account_region=config.deployment_account_region,
region=region,
deployment_account_role=deployment_account_role,
cloudformation=cloudformation)
threads = []
account_ids = organizations.get_account_ids()
for account_id in account_ids:
t = PropagatingThread(target=worker_thread,
args=(account_id, sts, config, s3, cache))
t.start()
threads.append(t)
for thread in threads:
thread.join()
step_functions = StepFunctions(
role=deployment_account_role,
deployment_account_id=deployment_account_id,
deployment_account_region=config.deployment_account_region,
regions=config.target_regions,
account_ids=[
i for i in account_ids if i != config.deployment_account_region
],
update_pipelines_only=False)
step_functions.execute_statemachine()
except NotConfiguredError as not_configured_error:
LOGGER.info(not_configured_error)
def main(): #pylint: disable=R0915
LOGGER.info("ADF Version %s", ADF_VERSION)
LOGGER.info("ADF Log Level is %s", ADF_LOG_LEVEL)
scp = SCP()
config = Config()
config.store_config()
try:
parameter_store = ParameterStore(REGION_DEFAULT, boto3)
deployment_account_id = parameter_store.fetch_parameter(
'deployment_account_id'
)
organizations = Organizations(
role=boto3,
account_id=deployment_account_id
)
scp.apply(organizations, parameter_store, config.config)
sts = STS()
deployment_account_role = prepare_deployment_account(
sts=sts,
deployment_account_id=deployment_account_id,
config=config
)
cache = Cache()
ou_id = organizations.get_parent_info().get("ou_parent_id")
account_path = organizations.build_account_path(
ou_id=ou_id,
account_path=[],
cache=cache
)
s3 = S3(
region=REGION_DEFAULT,
bucket=S3_BUCKET_NAME
)
# Updating the stack on the master account in deployment region
cloudformation = CloudFormation(
region=config.deployment_account_region,
deployment_account_region=config.deployment_account_region, # pylint: disable=R0801
role=boto3,
wait=True,
stack_name=None,
s3=s3,
s3_key_path='adf-build',
account_id=ACCOUNT_ID
)
cloudformation.create_stack()
# First Setup/Update the Deployment Account in all regions (KMS Key and S3 Bucket + Parameter Store values)
for region in list(set([config.deployment_account_region] + config.target_regions)):
cloudformation = CloudFormation(
region=region,
deployment_account_region=config.deployment_account_region,
role=deployment_account_role,
wait=True,
stack_name=None,
s3=s3,
s3_key_path=account_path,
account_id=deployment_account_id
)
cloudformation.create_stack()
update_deployment_account_output_parameters(
deployment_account_region=config.deployment_account_region,
region=region,
deployment_account_role=deployment_account_role,
cloudformation=cloudformation
)
threads = []
account_ids = organizations.get_account_ids()
for account_id in [account for account in account_ids if account != deployment_account_id]:
thread = PropagatingThread(target=worker_thread, args=(
account_id,
sts,
config,
s3,
cache
))
thread.start()
threads.append(thread)
for thread in threads:
thread.join()
step_functions = StepFunctions(
role=deployment_account_role,
deployment_account_id=deployment_account_id,
deployment_account_region=config.deployment_account_region,
regions=config.target_regions,
account_ids=account_ids,
update_pipelines_only=0
)
step_functions.execute_statemachine()
except ParameterNotFoundError:
LOGGER.info(
'You are now ready to bootstrap a deployment account '
'by moving it into your deployment OU. '
'Once you have moved it into the deployment OU, '
'be sure to check out its progress in AWS Step Functions'
)
return