def main():
parameter_store = ParameterStore(DEPLOYMENT_ACCOUNT_REGION, boto3)
deployment_map = DeploymentMap(parameter_store,
os.environ["ADF_PIPELINE_PREFIX"])
s3 = S3(DEPLOYMENT_ACCOUNT_REGION, boto3, S3_BUCKET_NAME)
sts = STS(boto3)
role = sts.assume_cross_account_role(
'arn:aws:iam::{0}:role/{1}-org-access-adf'.format(
MASTER_ACCOUNT_ID,
parameter_store.fetch_parameter('cross_account_access_role')),
'pipeline')
organizations = Organizations(role)
clean(parameter_store, deployment_map)
for p in deployment_map.map_contents.get('pipelines'):
pipeline = Pipeline(p)
for target in p['targets']:
target_structure = TargetStructure(target)
for step in target_structure.target:
for path in step.get('path'):
try:
regions = step.get(
'regions',
p.get('regions', DEPLOYMENT_ACCOUNT_REGION))
pipeline.stage_regions.append(regions)
pipeline_target = Target(path, regions,
target_structure,
organizations)
pipeline_target.fetch_accounts_for_target()
except BaseException:
raise Exception(
"Failed to return accounts for {0}".format(path))
pipeline.template_dictionary["targets"].append(
target_structure.account_list)
if DEPLOYMENT_ACCOUNT_REGION not in regions:
pipeline.stage_regions.append(DEPLOYMENT_ACCOUNT_REGION)
parameters = pipeline.generate_parameters()
pipeline.generate()
deployment_map.update_deployment_parameters(pipeline)
s3_object_path = upload_if_required(s3, pipeline)
store_regional_parameter_config(pipeline, parameter_store)
cloudformation = CloudFormation(
region=DEPLOYMENT_ACCOUNT_REGION,
deployment_account_region=DEPLOYMENT_ACCOUNT_REGION,
role=boto3,
template_url=s3_object_path,
parameters=parameters,
wait=True,
stack_name="{0}-{1}".format(os.environ["ADF_PIPELINE_PREFIX"],
pipeline.name),
s3=None,
s3_key_path=None)
cloudformation.create_stack()
def main(): #pylint: disable=R0915
LOGGER.info('ADF Version %s', ADF_VERSION)
LOGGER.info("ADF Log Level is %s", ADF_LOG_LEVEL)
parameter_store = ParameterStore(DEPLOYMENT_ACCOUNT_REGION, boto3)
deployment_map = DeploymentMap(parameter_store, ADF_PIPELINE_PREFIX)
s3 = S3(DEPLOYMENT_ACCOUNT_REGION, S3_BUCKET_NAME)
sts = STS()
role = sts.assume_cross_account_role(
'arn:aws:iam::{0}:role/{1}-readonly'.format(
MASTER_ACCOUNT_ID,
parameter_store.fetch_parameter('cross_account_access_role')),
'pipeline')
organizations = Organizations(role)
clean(parameter_store, deployment_map)
for p in deployment_map.map_contents.get('pipelines'):
pipeline = Pipeline(p)
for target in p.get('targets', []):
target_structure = TargetStructure(target)
for step in target_structure.target:
for path in step.get('path'):
regions = step.get(
'regions', p.get('regions', DEPLOYMENT_ACCOUNT_REGION))
step_name = step.get('name')
params = step.get('params', {})
pipeline.stage_regions.append(regions)
pipeline_target = Target(path, regions, target_structure,
organizations, step_name, params)
pipeline_target.fetch_accounts_for_target()
pipeline.template_dictionary["targets"].append(
target_structure.account_list)
if DEPLOYMENT_ACCOUNT_REGION not in regions:
pipeline.stage_regions.append(DEPLOYMENT_ACCOUNT_REGION)
parameters = pipeline.generate_parameters()
pipeline.generate()
deployment_map.update_deployment_parameters(pipeline)
s3_object_path = upload_pipeline(s3, pipeline)
store_regional_parameter_config(pipeline, parameter_store)
cloudformation = CloudFormation(
region=DEPLOYMENT_ACCOUNT_REGION,
deployment_account_region=DEPLOYMENT_ACCOUNT_REGION,
role=boto3,
template_url=s3_object_path,
parameters=parameters,
wait=True,
stack_name="{0}-{1}".format(ADF_PIPELINE_PREFIX, pipeline.name),
s3=None,
s3_key_path=None,
account_id=DEPLOYMENT_ACCOUNT_ID)
cloudformation.create_stack()
def main():
LOGGER.info('ADF Version %s', ADF_VERSION)
LOGGER.info("ADF Log Level is %s", ADF_LOG_LEVEL)
parameter_store = ParameterStore(DEPLOYMENT_ACCOUNT_REGION, boto3)
deployment_map = DeploymentMap(parameter_store, ADF_PIPELINE_PREFIX)
s3 = S3(DEPLOYMENT_ACCOUNT_REGION, S3_BUCKET_NAME)
sts = STS()
role = sts.assume_cross_account_role(
'arn:aws:iam::{0}:role/{1}-readonly'.format(
MASTER_ACCOUNT_ID,
parameter_store.fetch_parameter('cross_account_access_role')),
'pipeline')
organizations = Organizations(role)
clean(parameter_store, deployment_map)
try:
auto_create_repositories = parameter_store.fetch_parameter(
'auto_create_repositories')
except ParameterNotFoundError:
auto_create_repositories = 'enabled'
threads = []
for p in deployment_map.map_contents.get('pipelines'):
thread = PropagatingThread(target=worker_thread,
args=(p, organizations,
auto_create_repositories, s3,
deployment_map, parameter_store))
thread.start()
threads.append(thread)
for thread in threads:
thread.join()
def cls():
return DeploymentMap(
parameter_store=None,
pipeline_name_prefix='adf',
map_path='{0}/stubs/stub_deployment_map.yml'.format(
os.path.dirname(os.path.realpath(__file__))
)
)
def main():
LOGGER.info('ADF Version %s', ADF_VERSION)
LOGGER.info("ADF Log Level is %s", ADF_LOG_LEVEL)
parameter_store = ParameterStore(DEPLOYMENT_ACCOUNT_REGION, boto3)
s3 = S3(DEPLOYMENT_ACCOUNT_REGION, SHARED_MODULES_BUCKET)
deployment_map = DeploymentMap(parameter_store, s3, ADF_PIPELINE_PREFIX)
LOGGER.info('Cleaning Stale Deployment Map entries')
clean(parameter_store, deployment_map)
def main():
LOGGER.info('ADF Version %s', ADF_VERSION)
LOGGER.info("ADF Log Level is %s", ADF_LOG_LEVEL)
parameter_store = ParameterStore(
DEPLOYMENT_ACCOUNT_REGION,
boto3
)
deployment_map = DeploymentMap(
parameter_store,
ADF_PIPELINE_PREFIX
)
s3 = S3(
DEPLOYMENT_ACCOUNT_REGION,
S3_BUCKET_NAME
)
sts = STS()
role = sts.assume_cross_account_role(
'arn:aws:iam::{0}:role/{1}-readonly'.format(
MASTER_ACCOUNT_ID,
parameter_store.fetch_parameter('cross_account_access_role')
), 'pipeline'
)
organizations = Organizations(role)
clean(parameter_store, deployment_map)
try:
auto_create_repositories = parameter_store.fetch_parameter('auto_create_repositories')
except ParameterNotFoundError:
auto_create_repositories = 'enabled'
threads = []
for counter, p in enumerate(deployment_map.map_contents.get('pipelines')):
thread = PropagatingThread(target=worker_thread, args=(
p,
organizations,
auto_create_repositories,
s3,
deployment_map,
parameter_store
))
thread.start()
threads.append(thread)
_batcher = counter % 10
if _batcher == 9: # 9 meaning we have hit a set of 10 threads since n % 10
_interval = random.randint(5, 11)
LOGGER.debug('Waiting for %s seconds before starting next batch of 10 threads.', _interval)
time.sleep(_interval)
for thread in threads:
thread.join()
def main():
LOGGER.info('ADF Version %s', ADF_VERSION)
LOGGER.info("ADF Log Level is %s", ADF_LOG_LEVEL)
_create_inputs_folder()
parameter_store = ParameterStore(
DEPLOYMENT_ACCOUNT_REGION,
boto3
)
s3 = S3(DEPLOYMENT_ACCOUNT_REGION, SHARED_MODULES_BUCKET)
deployment_map = DeploymentMap(
parameter_store,
s3,
ADF_PIPELINE_PREFIX
)
sts = STS()
role = sts.assume_cross_account_role(
'arn:aws:iam::{0}:role/{1}-readonly'.format(
MASTER_ACCOUNT_ID,
parameter_store.fetch_parameter('cross_account_access_role')
), 'pipeline'
)
organizations = Organizations(role)
clean(parameter_store, deployment_map)
ensure_event_bus_status(ORGANIZATION_ID)
try:
auto_create_repositories = parameter_store.fetch_parameter('auto_create_repositories')
except ParameterNotFoundError:
auto_create_repositories = 'enabled'
threads = []
_cache = Cache()
for p in deployment_map.map_contents.get('pipelines', []):
_source_account_id = p.get('default_providers', {}).get('source', {}).get('properties', {}).get('account_id', {})
if _source_account_id and int(_source_account_id) != int(DEPLOYMENT_ACCOUNT_ID) and not _cache.check(_source_account_id):
rule = Rule(p['default_providers']['source']['properties']['account_id'])
rule.create_update()
_cache.add(p['default_providers']['source']['properties']['account_id'], True)
thread = PropagatingThread(target=worker_thread, args=(
p,
organizations,
auto_create_repositories,
deployment_map,
parameter_store
))
thread.start()
threads.append(thread)
for thread in threads:
thread.join()