def generate_tasks(f, single_account=None, is_dry_run=False):
puppet_account_id = config.get_puppet_account_id()
manifest = manifest_utils.load(f)
should_use_sns = config.get_should_use_sns(os.environ.get("AWS_DEFAULT_REGION"))
should_use_product_plans = config.get_should_use_product_plans(os.environ.get("AWS_DEFAULT_REGION"))
tasks_to_run = manifest_utils_for_launches.generate_launch_tasks(
manifest,
puppet_account_id,
should_use_sns,
should_use_product_plans,
include_expanded_from=False,
single_account=single_account,
is_dry_run=is_dry_run,
)
logger.info("Finished generating provisioning tasks")
if not is_dry_run:
logger.info("Generating sharing tasks")
spoke_local_portfolios_tasks = manifest_utils_for_spoke_local_portfolios.generate_spoke_local_portfolios_tasks(
manifest,
puppet_account_id,
should_use_sns,
should_use_product_plans,
include_expanded_from=False,
single_account=single_account,
is_dry_run=is_dry_run,
)
tasks_to_run += spoke_local_portfolios_tasks
logger.info("Finished generating sharing tasks")
logger.info("Finished generating all tasks")
return tasks_to_run
def deploy(
f,
puppet_account_id,
executor_account_id,
single_account=None,
num_workers=10,
is_dry_run=False,
is_list_launches=False,
execution_mode="hub",
on_complete_url=None,
running_exploded=False,
):
os.environ["SCT_CACHE_INVALIDATOR"] = str(datetime.now())
os.environ["SCT_EXECUTION_MODE"] = str(execution_mode)
os.environ["SCT_SINGLE_ACCOUNT"] = str(single_account)
os.environ["SCT_IS_DRY_RUN"] = str(is_dry_run)
os.environ["SCT_SHOULD_USE_SNS"] = str(
config.get_should_use_sns(puppet_account_id))
os.environ["SCT_SHOULD_USE_PRODUCT_PLANS"] = str(
config.get_should_use_product_plans(
puppet_account_id, os.environ.get("AWS_DEFAULT_REGION")))
tasks_to_run = generate_tasks(f, puppet_account_id, executor_account_id,
execution_mode, is_dry_run)
runner.run_tasks(
puppet_account_id,
executor_account_id,
tasks_to_run,
num_workers,
is_dry_run,
is_list_launches,
execution_mode,
on_complete_url,
running_exploded,
)
def generate_tasks(
f,
puppet_account_id,
executor_account_id,
single_account=None,
is_dry_run=False,
execution_mode="hub",
cache_invalidator="now",
):
should_use_sns = config.get_should_use_sns(
puppet_account_id, os.environ.get("AWS_DEFAULT_REGION")
)
should_use_product_plans = config.get_should_use_product_plans(
puppet_account_id, os.environ.get("AWS_DEFAULT_REGION")
)
return [
launch_tasks.LaunchSectionTask(
manifest_file_path=f.name,
puppet_account_id=puppet_account_id,
should_use_sns=should_use_sns,
should_use_product_plans=should_use_product_plans,
include_expanded_from=False,
single_account=single_account,
is_dry_run=is_dry_run,
execution_mode=execution_mode,
cache_invalidator=cache_invalidator,
),
spoke_local_portfolios_tasks.SpokeLocalPortfolioSectionTask(
manifest_file_path=f.name,
puppet_account_id=puppet_account_id,
should_use_sns=should_use_sns,
should_use_product_plans=should_use_product_plans,
include_expanded_from=False,
single_account=single_account,
is_dry_run=is_dry_run,
execution_mode=execution_mode,
cache_invalidator=cache_invalidator,
),
lambda_invocations_tasks.LambdaInvocationsSectionTask(
manifest_file_path=f.name,
puppet_account_id=puppet_account_id,
should_use_sns=should_use_sns,
should_use_product_plans=should_use_product_plans,
include_expanded_from=False,
single_account=single_account,
is_dry_run=is_dry_run,
execution_mode=execution_mode,
cache_invalidator=cache_invalidator,
),
]
def deploy(
f,
puppet_account_id,
executor_account_id,
single_account=None,
num_workers=10,
is_dry_run=False,
is_list_launches=False,
execution_mode="hub",
on_complete_url=None,
running_exploded=False,
output_cache_starting_point="",
):
if os.environ.get("SCT_CACHE_INVALIDATOR"):
logger.info(
f"Found existing SCT_CACHE_INVALIDATOR: {os.environ.get('SCT_CACHE_INVALIDATOR')}"
)
else:
os.environ["SCT_CACHE_INVALIDATOR"] = str(datetime.now())
os.environ["SCT_EXECUTION_MODE"] = str(execution_mode)
os.environ["SCT_SINGLE_ACCOUNT"] = str(single_account)
os.environ["SCT_IS_DRY_RUN"] = str(is_dry_run)
os.environ["EXECUTOR_ACCOUNT_ID"] = str(executor_account_id)
os.environ["SCT_SHOULD_USE_SNS"] = str(
config.get_should_use_sns(puppet_account_id))
os.environ["SCT_SHOULD_DELETE_ROLLBACK_COMPLETE_STACKS"] = str(
config.get_should_delete_rollback_complete_stacks(puppet_account_id))
os.environ["SCT_SHOULD_USE_PRODUCT_PLANS"] = str(
config.get_should_use_product_plans(
puppet_account_id, os.environ.get("AWS_DEFAULT_REGION")))
os.environ["SCT_INITIALISER_STACK_TAGS"] = json.dumps(
config.get_initialiser_stack_tags())
tasks_to_run = generate_tasks(f, puppet_account_id, executor_account_id,
execution_mode, is_dry_run)
runner.run_tasks(
puppet_account_id,
executor_account_id,
tasks_to_run,
num_workers,
is_dry_run,
is_list_launches,
execution_mode,
on_complete_url,
running_exploded,
output_cache_starting_point=output_cache_starting_point,
)
def generate_tasks(f, single_account=None, dry_run=False):
puppet_account_id = config.get_puppet_account_id()
manifest = manifest_utils.load(f)
tasks_to_run = []
should_use_sns = config.get_should_use_sns(
os.environ.get("AWS_DEFAULT_REGION"))
should_use_product_plans = config.get_should_use_product_plans(
os.environ.get("AWS_DEFAULT_REGION"))
task_defs = manifest_utils.convert_manifest_into_task_defs_for_launches(
manifest, puppet_account_id, should_use_sns, should_use_product_plans)
for task in task_defs:
if single_account is not None:
if task.get('account_id') != single_account:
continue
task_status = task.get('status')
del task['status']
if task_status == constants.PROVISIONED:
task['should_use_sns'] = should_use_sns
if dry_run:
tasks_to_run.append(
provisioning_tasks.ProvisionProductDryRunTask(**task))
else:
tasks_to_run.append(
provisioning_tasks.ProvisionProductTask(**task))
elif task_status == constants.TERMINATED:
for attribute in constants.DISALLOWED_ATTRIBUTES_FOR_TERMINATED_LAUNCHES:
logger.info(
f"checking {task.get('launch_name')} for disallowed attributes"
)
attribute_value = task.get(attribute)
if attribute_value is not None:
if isinstance(attribute_value, list):
if len(attribute_value) != 0:
raise Exception(
f"Launch {task.get('launch_name')} has disallowed attribute: {attribute}"
)
elif isinstance(attribute_value, dict):
if len(attribute_value.keys()) != 0:
raise Exception(
f"Launch {task.get('launch_name')} has disallowed attribute: {attribute}"
)
else:
raise Exception(
f"Launch {task.get('launch_name')} has disallowed attribute: {attribute}"
)
del task['launch_parameters']
del task['manifest_parameters']
del task['account_parameters']
del task['should_use_sns']
del task['requested_priority']
del task['should_use_product_plans']
del task['pre_actions']
del task['post_actions']
if dry_run:
tasks_to_run.append(
provisioning_tasks.TerminateProductDryRunTask(**task))
else:
tasks_to_run.append(
provisioning_tasks.TerminateProductTask(**task))
else:
raise Exception(f"Unsupported status of {task_status}")
if not dry_run:
spoke_local_portfolios_tasks = manifest_utils.convert_manifest_into_task_defs_for_spoke_local_portfolios(
manifest, puppet_account_id, should_use_sns, tasks_to_run)
for spoke_local_portfolios_task in spoke_local_portfolios_tasks:
if single_account is not None:
param_kwargs = spoke_local_portfolios_task.param_kwargs
logger.info(f"EPF:: {param_kwargs}")
if param_kwargs.get('account_id',
'not_an_account_id') != single_account:
continue
tasks_to_run.append(spoke_local_portfolios_task)
return tasks_to_run
def expand(f, puppet_account_id, single_account, subset=None):
click.echo("Expanding")
manifest = manifest_utils.load(f, puppet_account_id)
org_iam_role_arn = config.get_org_iam_role_arn(puppet_account_id)
if org_iam_role_arn is None:
click.echo("No org role set - not expanding")
new_manifest = manifest
else:
click.echo("Expanding using role: {}".format(org_iam_role_arn))
with betterboto_client.CrossAccountClientContextManager(
"organizations", org_iam_role_arn, "org-iam-role"
) as client:
new_manifest = manifest_utils.expand_manifest(manifest, client)
click.echo("Expanded")
if single_account:
click.echo(f"Filtering for single account: {single_account}")
for account in new_manifest.get("accounts", []):
if str(account.get("account_id")) == str(single_account):
click.echo(f"Found single account: {single_account}")
new_manifest["accounts"] = [account]
break
click.echo("Filtered")
new_manifest = manifest_utils.rewrite_depends_on(new_manifest)
new_manifest = manifest_utils.rewrite_ssm_parameters(new_manifest)
new_manifest = manifest_utils.rewrite_stacks(new_manifest, puppet_account_id)
if subset:
click.echo(f"Filtering for subset: {subset}")
new_manifest = manifest_utils.isolate(
manifest_utils.Manifest(new_manifest), subset
)
new_manifest = json.loads(json.dumps(new_manifest))
if new_manifest.get(constants.LAMBDA_INVOCATIONS) is None:
new_manifest[constants.LAMBDA_INVOCATIONS] = dict()
home_region = config.get_home_region(puppet_account_id)
with betterboto_client.ClientContextManager("ssm") as ssm:
response = ssm.get_parameter(Name="service-catalog-puppet-version")
version = response.get("Parameter").get("Value")
new_manifest["config_cache"] = dict(
home_region=home_region,
regions=config.get_regions(puppet_account_id, home_region),
should_collect_cloudformation_events=config.get_should_use_sns(
puppet_account_id, home_region
),
should_forward_events_to_eventbridge=config.get_should_use_eventbridge(
puppet_account_id, home_region
),
should_forward_failures_to_opscenter=config.get_should_forward_failures_to_opscenter(
puppet_account_id, home_region
),
puppet_version=version,
)
new_name = f.name.replace(".yaml", "-expanded.yaml")
logger.info("Writing new manifest: {}".format(new_name))
with open(new_name, "w") as output:
output.write(yaml.safe_dump(new_manifest, default_flow_style=False))
def expand(f, puppet_account_id, single_account, subset=None):
click.echo("Expanding")
target_directory = os.path.sep.join([os.path.dirname(f.name), "manifests"])
assemble_manifest_from_ssm(target_directory)
manifest = manifest_utils.load(f, puppet_account_id)
org_iam_role_arn = config.get_org_iam_role_arn(puppet_account_id)
if org_iam_role_arn is None:
click.echo("No org role set - not expanding")
new_manifest = manifest
else:
click.echo("Expanding using role: {}".format(org_iam_role_arn))
with betterboto_client.CrossAccountClientContextManager(
"organizations", org_iam_role_arn, "org-iam-role"
) as client:
new_manifest = manifest_utils.expand_manifest(manifest, client)
click.echo("Expanded")
new_manifest = manifest_utils.rewrite_deploy_as_share_to_for_spoke_local_portfolios(
new_manifest
)
if single_account:
click.echo(f"Filtering for single account: {single_account}")
for account in new_manifest.get("accounts", []):
if str(account.get("account_id")) == str(single_account):
click.echo(f"Found single account: {single_account}")
new_manifest["accounts"] = [account]
break
items_to_delete = list()
for section_name in constants.ALL_SECTION_NAMES:
deploy_to_name = constants.DEPLOY_TO_NAMES[section_name]
for item_name, item in new_manifest.get(section_name, {}).items():
accounts = list()
for deploy_details in item.get(deploy_to_name, {}).get("accounts", []):
if str(deploy_details.get("account_id")) == str(single_account):
accounts.append(deploy_details)
print(f"{item_name}: there are " + str(len(accounts)))
if item.get(deploy_to_name).get("accounts"):
if len(accounts) > 0:
item[deploy_to_name]["accounts"] = accounts
else:
if item[deploy_to_name].get("tags") or item[deploy_to_name].get(
"ous"
):
del item[deploy_to_name]["accounts"]
else:
items_to_delete.append(f"{section_name}:{item_name}")
for item_to_delete in items_to_delete:
section_name, item_name = item_to_delete.split(":")
del new_manifest[section_name][item_name]
click.echo("Filtered")
new_manifest = manifest_utils.rewrite_cfct(new_manifest)
new_manifest = manifest_utils.rewrite_depends_on(new_manifest)
new_manifest = manifest_utils.rewrite_ssm_parameters(new_manifest)
new_manifest = manifest_utils.rewrite_stacks(new_manifest, puppet_account_id)
new_manifest = manifest_utils.rewrite_scps(new_manifest, puppet_account_id)
new_manifest = manifest_utils.parse_conditions(new_manifest)
if subset and subset.get("section"):
click.echo(f"Filtering for subset: {subset}")
new_manifest = manifest_utils.isolate(new_manifest, subset)
manifest_accounts_all = [
{"account_id": a.get("account_id"), "email": a.get("email")}
for a in new_manifest.get("accounts", [])
]
manifest_accounts_excluding = [
a for a in manifest_accounts_all if a.get("account_id") != puppet_account_id
]
# handle all accounts
sct_manifest_accounts = json.dumps(manifest_accounts_all)
sct_manifest_spokes = json.dumps(manifest_accounts_excluding)
regions = config.get_regions(puppet_account_id)
sct_config_regions = json.dumps(regions)
new_manifest["parameters"]["SCTManifestAccounts"] = dict(
default=sct_manifest_accounts
)
new_manifest["parameters"]["SCTManifestSpokes"] = dict(default=sct_manifest_spokes)
new_manifest["parameters"]["SCTConfigRegions"] = dict(default=sct_config_regions)
new_manifest["parameters"]["SCTAccountId"] = dict(default=puppet_account_id)
if new_manifest.get(constants.LAMBDA_INVOCATIONS) is None:
new_manifest[constants.LAMBDA_INVOCATIONS] = dict()
home_region = config.get_home_region(puppet_account_id)
with betterboto_client.ClientContextManager("ssm") as ssm:
response = ssm.get_parameter(Name="service-catalog-puppet-version")
version = response.get("Parameter").get("Value")
new_manifest["config_cache"] = dict(
home_region=home_region,
regions=regions,
should_collect_cloudformation_events=config.get_should_use_sns(
puppet_account_id, home_region
),
should_forward_events_to_eventbridge=config.get_should_use_eventbridge(
puppet_account_id, home_region
),
should_forward_failures_to_opscenter=config.get_should_forward_failures_to_opscenter(
puppet_account_id, home_region
),
puppet_version=version,
)
new_name = f.name.replace(".yaml", "-expanded.yaml")
logger.info("Writing new manifest: {}".format(new_name))
with open(new_name, "w") as output:
output.write(yaml_utils.dump(new_manifest))
def run_tasks_for_generate_shares(tasks_to_run):
for type in [
"failure",
"success",
"timeout",
"process_failure",
"processing_time",
"broken_task",
]:
os.makedirs(Path(constants.RESULTS_DIRECTORY) / type)
run_result = luigi.build(
tasks_to_run,
local_scheduler=True,
detailed_summary=True,
workers=10,
log_level='INFO',
)
should_use_sns = config.get_should_use_sns()
puppet_account_id = config.get_puppet_account_id()
version = config.get_puppet_version()
for region in config.get_regions():
sharing_policies = {
'accounts': [],
'organizations': [],
}
with betterboto_client.ClientContextManager(
'cloudformation', region_name=region) as cloudformation:
cloudformation.ensure_deleted(
StackName="servicecatalog-puppet-shares")
logger.info(f"generating policies collection for region {region}")
if os.path.exists(os.path.sep.join(['data', 'bucket'])):
logger.info(f"Updating policies for the region: {region}")
path = os.path.sep.join(['data', 'bucket', region, 'accounts'])
if os.path.exists(path):
for account_file in os.listdir(path):
account = account_file.split(".")[0]
sharing_policies['accounts'].append(account)
path = os.path.sep.join(
['data', 'bucket', region, 'organizations'])
if os.path.exists(path):
for organization_file in os.listdir(path):
organization = organization_file.split(".")[0]
sharing_policies['organizations'].append(organization)
logger.info(f"Finished generating policies collection")
template = config.env.get_template(
'policies.template.yaml.j2').render(
sharing_policies=sharing_policies,
VERSION=version,
)
with betterboto_client.ClientContextManager(
'cloudformation', region_name=region) as cloudformation:
cloudformation.create_or_update(
StackName="servicecatalog-puppet-policies",
TemplateBody=template,
NotificationARNs=[
f"arn:aws:sns:{region}:{puppet_account_id}:servicecatalog-puppet-cloudformation-regional-events"
] if should_use_sns else [],
)
for filename in glob('results/failure/*.json'):
result = json.loads(open(filename, 'r').read())
click.echo(
colorclass.Color("{red}" + result.get('task_type') +
" failed{/red}"))
click.echo(
f"{yaml.safe_dump({'parameters':result.get('task_params')})}")
click.echo("\n".join(result.get('exception_stack_trace')))
click.echo('')
exit_status_codes = {
LuigiStatusCode.SUCCESS: 0,
LuigiStatusCode.SUCCESS_WITH_RETRY: 0,
LuigiStatusCode.FAILED: 1,
LuigiStatusCode.FAILED_AND_SCHEDULING_FAILED: 2,
LuigiStatusCode.SCHEDULING_FAILED: 3,
LuigiStatusCode.NOT_RUN: 4,
LuigiStatusCode.MISSING_EXT: 5,
}
sys.exit(exit_status_codes.get(run_result.status))
