def reset_provisioned_product_owner(f):
puppet_account_id = config.get_puppet_account_id()
current_account_id = puppet_account_id
manifest = manifest_utils.load(f, puppet_account_id)
os.environ["SCT_CACHE_INVALIDATOR"] = str(datetime.now())
task_defs = manifest_utils_for_launches.generate_launch_tasks(
manifest, puppet_account_id, False, False)
tasks_to_run = []
for task in task_defs:
task_status = task.get("status")
if task_status == constants.PROVISIONED:
tasks_to_run.append(
launch_tasks.ResetProvisionedProductOwnerTask(
launch_name=task.get("launch_name"),
account_id=task.get("account_id"),
region=task.get("region"),
))
runner.run_tasks(
puppet_account_id,
current_account_id,
tasks_to_run,
10,
execution_mode="hub",
on_complete_url=None,
)
def dry_run(f):
manifest = manifest_utils.load(f)
launch_tasks = {}
tasks_to_run = []
all_launch_tasks = cli_command_helpers.deploy_launches(manifest)
launch_tasks.update(all_launch_tasks)
for task in cli_command_helpers.wire_dependencies(launch_tasks):
task_status = task.get('status')
del task['status']
if task_status == constants.PROVISIONED:
tasks_to_run.append(
luigi_tasks_and_targets.ProvisionProductDryRunTask(**task))
elif task_status == constants.TERMINATED:
tasks_to_run.append(
luigi_tasks_and_targets.TerminateProductDryRunTask(**task))
else:
raise Exception(f"Unsupported status of {task_status}")
# spoke_local_portfolio_tasks_to_run = cli_command_helpers.deploy_spoke_local_portfolios(manifest, launch_tasks)
# tasks_to_run += spoke_local_portfolio_tasks_to_run
cli_command_helpers.run_tasks_for_dry_run(tasks_to_run)
def reset_provisioned_product_owner(f):
puppet_account_id = config.get_puppet_account_id()
current_account_id = puppet_account_id
manifest = manifest_utils.load(f, puppet_account_id)
task_defs = manifest_utils_for_launches.generate_launch_tasks(
manifest, puppet_account_id, False, False
)
tasks_to_run = []
for task in task_defs:
task_status = task.get("status")
if task_status == constants.PROVISIONED:
tasks_to_run.append(
provisioning_tasks.ResetProvisionedProductOwnerTask(
launch_name=task.get("launch_name"),
account_id=task.get("account_id"),
region=task.get("region"),
)
)
cache_invalidator = str(datetime.now())
runner.run_tasks(
puppet_account_id,
current_account_id,
tasks_to_run,
10,
cache_invalidator=cache_invalidator,
on_complete_url=None,
)
def expand(f, single_account):
click.echo("Expanding")
puppet_account_id = config.get_puppet_account_id()
manifest = manifest_utils.load(f, puppet_account_id)
org_iam_role_arn = config.get_org_iam_role_arn(puppet_account_id)
if org_iam_role_arn is None:
click.echo("No org role set - not expanding")
new_manifest = manifest
else:
click.echo("Expanding using role: {}".format(org_iam_role_arn))
with betterboto_client.CrossAccountClientContextManager(
"organizations", org_iam_role_arn, "org-iam-role"
) as client:
new_manifest = manifest_utils.expand_manifest(manifest, client)
click.echo("Expanded")
if single_account:
click.echo(f"Filtering for single account: {single_account}")
for account in new_manifest.get("accounts", []):
if account.get("account_id") == single_account:
click.echo(f"Found single account: {single_account}")
new_manifest["accounts"] = [account]
break
click.echo("Filtered")
new_name = f.name.replace(".yaml", "-expanded.yaml")
logger.info("Writing new manifest: {}".format(new_name))
with open(new_name, "w") as output:
output.write(yaml.safe_dump(new_manifest, default_flow_style=False))
def generate_tasks(f, single_account=None, is_dry_run=False):
puppet_account_id = config.get_puppet_account_id()
manifest = manifest_utils.load(f)
should_use_sns = config.get_should_use_sns(os.environ.get("AWS_DEFAULT_REGION"))
should_use_product_plans = config.get_should_use_product_plans(os.environ.get("AWS_DEFAULT_REGION"))
tasks_to_run = manifest_utils_for_launches.generate_launch_tasks(
manifest,
puppet_account_id,
should_use_sns,
should_use_product_plans,
include_expanded_from=False,
single_account=single_account,
is_dry_run=is_dry_run,
)
logger.info("Finished generating provisioning tasks")
if not is_dry_run:
logger.info("Generating sharing tasks")
spoke_local_portfolios_tasks = manifest_utils_for_spoke_local_portfolios.generate_spoke_local_portfolios_tasks(
manifest,
puppet_account_id,
should_use_sns,
should_use_product_plans,
include_expanded_from=False,
single_account=single_account,
is_dry_run=is_dry_run,
)
tasks_to_run += spoke_local_portfolios_tasks
logger.info("Finished generating sharing tasks")
logger.info("Finished generating all tasks")
return tasks_to_run
def deploy(f, single_account):
puppet_account_id = cli_command_helpers.get_puppet_account_id()
manifest = manifest_utils.load(f)
launch_tasks = {}
tasks_to_run = []
should_use_sns = cli_command_helpers.get_should_use_sns(
os.environ.get("AWS_DEFAULT_REGION"))
all_launch_tasks = cli_command_helpers.deploy_launches(
manifest, puppet_account_id)
launch_tasks.update(all_launch_tasks)
for task in cli_command_helpers.wire_dependencies(launch_tasks):
task_status = task.get('status')
del task['status']
if task_status == constants.PROVISIONED:
task['should_use_sns'] = should_use_sns
tasks_to_run.append(
luigi_tasks_and_targets.ProvisionProductTask(**task))
elif task_status == constants.TERMINATED:
for attribute in constants.DISALLOWED_ATTRIBUTES_FOR_TERMINATED_LAUNCHES:
logger.info(
f"checking {task.get('launch_name')} for disallowed attributes"
)
attribute_value = task.get(attribute)
if attribute_value is not None:
if isinstance(attribute_value, list):
if len(attribute_value) != 0:
raise Exception(
f"Launch {task.get('launch_name')} has disallowed attribute: {attribute}"
)
elif isinstance(attribute_value, dict):
if len(attribute_value.keys()) != 0:
raise Exception(
f"Launch {task.get('launch_name')} has disallowed attribute: {attribute}"
)
else:
raise Exception(
f"Launch {task.get('launch_name')} has disallowed attribute: {attribute}"
)
tasks_to_run.append(
luigi_tasks_and_targets.TerminateProductTask(**task))
else:
raise Exception(f"Unsupported status of {task_status}")
spoke_local_portfolio_tasks_to_run = cli_command_helpers.deploy_spoke_local_portfolios(
manifest, launch_tasks, should_use_sns, puppet_account_id)
tasks_to_run += spoke_local_portfolio_tasks_to_run
cli_command_helpers.run_tasks(tasks_to_run)
def generate_shares(f):
logger.info('Starting to generate shares for: {}'.format(f.name))
manifest = manifest_utils.load(f)
deployment_map = manifest_utils.build_deployment_map(
manifest, constants.LAUNCHES)
import_map = manifest_utils.build_deployment_map(
manifest, constants.SPOKE_LOCAL_PORTFOLIOS)
cli_command_helpers.create_share_template(
deployment_map, import_map,
cli_command_helpers.get_puppet_account_id())
def expand(f):
click.echo('Expanding')
manifest = manifest_utils.load(f)
org_iam_role_arn = cli_command_helpers.get_org_iam_role_arn()
if org_iam_role_arn is None:
click.echo('No org role set - not expanding')
new_manifest = manifest
else:
click.echo('Expanding using role: {}'.format(org_iam_role_arn))
with betterboto_client.CrossAccountClientContextManager(
'organizations', org_iam_role_arn, 'org-iam-role') as client:
new_manifest = manifest_utils.expand_manifest(manifest, client)
click.echo('Expanded')
new_name = f.name.replace(".yaml", '-expanded.yaml')
logger.info('Writing new manifest: {}'.format(new_name))
with open(new_name, 'w') as output:
output.write(yaml.safe_dump(new_manifest, default_flow_style=False))
def explode(f):
logger.info("Exploding")
puppet_account_id = config.get_puppet_account_id()
original_name = f.name
expanded_output = f.name.replace(".yaml", "-expanded.yaml")
expanded_manifest = manifest_utils.load(
open(expanded_output, "r"), puppet_account_id
)
expanded_manifest = manifest_utils.Manifest(expanded_manifest)
exploded = manifest_utils.explode(expanded_manifest)
logger.info(f"found {len(exploded)} graphs")
count = 0
for mani in exploded:
with open(original_name.replace(".yaml", f"-exploded-{count}.yaml"), "w") as f:
f.write(yaml.safe_dump(json.loads(json.dumps(mani))))
count += 1
def validate(f):
logger.info("Validating {}".format(f.name))
manifest = manifest_utils.load(f, config.get_puppet_account_id())
schema = yamale.make_schema(
asset_helpers.resolve_from_site_packages("schema.yaml"))
data = yamale.make_data(content=yaml.safe_dump(manifest))
yamale.validate(schema, data, strict=False)
tags_defined_by_accounts = {}
for account in manifest.get("accounts"):
for tag in account.get("tags", []):
tags_defined_by_accounts[tag] = True
for collection_type in constants.ALL_SECTION_NAMES:
collection_to_check = manifest.get(collection_type, {})
for collection_name, collection_item in collection_to_check.items():
for deploy_to in collection_item.get("deploy_to",
{}).get("tags", []):
tag_to_check = deploy_to.get("tag")
if tags_defined_by_accounts.get(tag_to_check) is None:
print(
f"{collection_type}.{collection_name} uses tag {tag_to_check} in deploy_to that does not exist"
)
for depends_on in collection_item.get("depends_on", []):
if isinstance(depends_on, str):
if manifest.get(
constants.LAUNCHES).get(depends_on) is None:
print(
f"{collection_type}.{collection_name} uses {depends_on} in depends_on that does not exist"
)
else:
tt = constants.SECTION_SINGULAR_TO_PLURAL.get(
depends_on.get("type", constants.LAUNCH))
dd = depends_on.get("name")
if manifest.get(tt).get(dd) is None:
print(
f"{collection_type}.{collection_name} uses {depends_on} in depends_on that does not exist"
)
click.echo("Finished validating: {}".format(f.name))
click.echo("Finished validating: OK")
def reset_provisioned_product_owner(f):
manifest = manifest_utils.load(f)
launch_tasks = {}
tasks_to_run = []
all_launch_tasks = cli_command_helpers.deploy_launches(manifest)
launch_tasks.update(all_launch_tasks)
for task in cli_command_helpers.wire_dependencies(launch_tasks):
task_status = task.get('status')
del task['status']
if task_status == constants.PROVISIONED:
tasks_to_run.append(
luigi_tasks_and_targets.ResetProvisionedProductOwnerTask(
**task))
cli_command_helpers.run_tasks(tasks_to_run)
def reset_provisioned_product_owner(f):
puppet_account_id = config.get_puppet_account_id()
manifest = manifest_utils.load(f)
task_defs = manifest_utils.convert_manifest_into_task_defs_for_launches(
manifest, puppet_account_id, False, False)
tasks_to_run = []
for task in task_defs:
task_status = task.get('status')
if task_status == constants.PROVISIONED:
tasks_to_run.append(
provisioning_tasks.ResetProvisionedProductOwnerTask(
launch_name=task.get('launch_name'),
account_id=task.get('account_id'),
region=task.get('region'),
))
runner.run_tasks(tasks_to_run, 10)
def expand(f, single_account, subset=None):
click.echo("Expanding")
puppet_account_id = config.get_puppet_account_id()
manifest = manifest_utils.load(f, puppet_account_id)
org_iam_role_arn = config.get_org_iam_role_arn(puppet_account_id)
if org_iam_role_arn is None:
click.echo("No org role set - not expanding")
new_manifest = manifest
else:
click.echo("Expanding using role: {}".format(org_iam_role_arn))
with betterboto_client.CrossAccountClientContextManager(
"organizations", org_iam_role_arn, "org-iam-role") as client:
new_manifest = manifest_utils.expand_manifest(manifest, client)
click.echo("Expanded")
if single_account:
click.echo(f"Filtering for single account: {single_account}")
for account in new_manifest.get("accounts", []):
if str(account.get("account_id")) == str(single_account):
click.echo(f"Found single account: {single_account}")
new_manifest["accounts"] = [account]
break
click.echo("Filtered")
new_manifest = manifest_utils.rewrite_depends_on(new_manifest)
if subset:
click.echo(f"Filtering for subset: {subset}")
new_manifest = manifest_utils.isolate(
manifest_utils.Manifest(new_manifest), subset)
new_manifest = json.loads(json.dumps(new_manifest))
if new_manifest.get(constants.LAMBDA_INVOCATIONS) is None:
new_manifest[constants.LAMBDA_INVOCATIONS] = dict()
new_name = f.name.replace(".yaml", "-expanded.yaml")
logger.info("Writing new manifest: {}".format(new_name))
with open(new_name, "w") as output:
output.write(yaml.safe_dump(new_manifest, default_flow_style=False))
def generate_shares(f):
logger.info('Starting to generate shares for: {}'.format(f.name))
tasks_to_run = []
puppet_account_id = config.get_puppet_account_id()
manifest = manifest_utils.load(f)
task_defs = manifest_utils.convert_manifest_into_task_defs_for_launches(
manifest, puppet_account_id, False, False, include_expanded_from=True)
for task in task_defs:
tasks_to_run.append(
portfoliomanagement_tasks.CreateShareForAccountLaunchRegion(
puppet_account_id=puppet_account_id,
account_id=task.get('account_id'),
region=task.get('region'),
portfolio=task.get('portfolio'),
expanded_from=task.get('expanded_from'),
organization=task.get('organization'),
))
spoke_local_portfolios_tasks = manifest_utils.convert_manifest_into_task_defs_for_spoke_local_portfolios(
manifest, puppet_account_id, False, tasks_to_run)
for task in spoke_local_portfolios_tasks:
if isinstance(task,
portfoliomanagement_tasks.CreateSpokeLocalPortfolioTask):
param_kwargs = task.param_kwargs
tasks_to_run.append(
portfoliomanagement_tasks.CreateShareForAccountLaunchRegion(
puppet_account_id=puppet_account_id,
account_id=param_kwargs.get('account_id'),
region=param_kwargs.get('region'),
portfolio=param_kwargs.get('portfolio'),
expanded_from=param_kwargs.get('expanded_from'),
organization=param_kwargs.get('organization'),
))
runner.run_tasks_for_generate_shares(tasks_to_run)
def validate(f):
logger.info("Validating {}".format(f.name))
manifest = manifest_utils.load(f, config.get_puppet_account_id())
try:
Loader = yaml.CSafeLoader
except AttributeError: # System does not have libyaml
Loader = yaml.SafeLoader
Loader.add_constructor("!Equals", yaml_utils.Equals.from_yaml)
Loader.add_constructor("!Not", yaml_utils.Not.from_yaml)
schema = yamale.make_schema(asset_helpers.resolve_from_site_packages("schema.yaml"))
data = yamale.make_data(content=yaml_utils.dump(manifest))
yamale.validate(schema, data, strict=False)
has_default_default_region = (
manifest.get("defaults", {}).get("accounts", {}).get("default_region", False)
)
has_default_regions_enabled = (
manifest.get("defaults", {}).get("accounts", {}).get("regions_enabled", False)
)
tags_defined_by_accounts = {}
for account in manifest.get("accounts"):
account_entry_is_an_overwrite_or_append = account.get(
"append", account.get("overwrite", False)
)
if (
not account_entry_is_an_overwrite_or_append
and account.get("default_region") is None
and not has_default_default_region
):
raise Exception(
f"account entry {account.get('account_id', account.get('ou'))} is missing default_region"
)
if (
not account_entry_is_an_overwrite_or_append
and account.get("regions_enabled") is None
and not has_default_regions_enabled
):
raise Exception(
f"account entry {account.get('account_id', account.get('ou'))} is missing regions_enabled"
)
for tag in account.get("tags", []):
tags_defined_by_accounts[tag] = True
for collection_type in constants.ALL_SECTION_NAMES:
collection_to_check = manifest.get(collection_type, {})
for collection_name, collection_item in collection_to_check.items():
for deploy_to in collection_item.get("deploy_to", {}).get("tags", []):
tag_to_check = deploy_to.get("tag")
if tags_defined_by_accounts.get(tag_to_check) is None:
print(
f"{collection_type}.{collection_name} uses tag {tag_to_check} in deploy_to that does not exist"
)
for depends_on in collection_item.get("depends_on", []):
if isinstance(depends_on, str):
if manifest.get(constants.LAUNCHES).get(depends_on) is None:
print(
f"{collection_type}.{collection_name} uses {depends_on} in depends_on that does not exist"
)
else:
tt = constants.SECTION_SINGULAR_TO_PLURAL.get(
depends_on.get("type", constants.LAUNCH)
)
dd = depends_on.get("name")
if manifest.get(tt).get(dd) is None:
print(
f"{collection_type}.{collection_name} uses {depends_on} in depends_on that does not exist"
)
click.echo("Finished validating: {}".format(f.name))
click.echo("Finished validating: OK")
def expand(f, puppet_account_id, single_account, subset=None):
click.echo("Expanding")
target_directory = os.path.sep.join([os.path.dirname(f.name), "manifests"])
assemble_manifest_from_ssm(target_directory)
manifest = manifest_utils.load(f, puppet_account_id)
org_iam_role_arn = config.get_org_iam_role_arn(puppet_account_id)
if org_iam_role_arn is None:
click.echo("No org role set - not expanding")
new_manifest = manifest
else:
click.echo("Expanding using role: {}".format(org_iam_role_arn))
with betterboto_client.CrossAccountClientContextManager(
"organizations", org_iam_role_arn, "org-iam-role"
) as client:
new_manifest = manifest_utils.expand_manifest(manifest, client)
click.echo("Expanded")
new_manifest = manifest_utils.rewrite_deploy_as_share_to_for_spoke_local_portfolios(
new_manifest
)
if single_account:
click.echo(f"Filtering for single account: {single_account}")
for account in new_manifest.get("accounts", []):
if str(account.get("account_id")) == str(single_account):
click.echo(f"Found single account: {single_account}")
new_manifest["accounts"] = [account]
break
items_to_delete = list()
for section_name in constants.ALL_SECTION_NAMES:
deploy_to_name = constants.DEPLOY_TO_NAMES[section_name]
for item_name, item in new_manifest.get(section_name, {}).items():
accounts = list()
for deploy_details in item.get(deploy_to_name, {}).get("accounts", []):
if str(deploy_details.get("account_id")) == str(single_account):
accounts.append(deploy_details)
print(f"{item_name}: there are " + str(len(accounts)))
if item.get(deploy_to_name).get("accounts"):
if len(accounts) > 0:
item[deploy_to_name]["accounts"] = accounts
else:
if item[deploy_to_name].get("tags") or item[deploy_to_name].get(
"ous"
):
del item[deploy_to_name]["accounts"]
else:
items_to_delete.append(f"{section_name}:{item_name}")
for item_to_delete in items_to_delete:
section_name, item_name = item_to_delete.split(":")
del new_manifest[section_name][item_name]
click.echo("Filtered")
new_manifest = manifest_utils.rewrite_cfct(new_manifest)
new_manifest = manifest_utils.rewrite_depends_on(new_manifest)
new_manifest = manifest_utils.rewrite_ssm_parameters(new_manifest)
new_manifest = manifest_utils.rewrite_stacks(new_manifest, puppet_account_id)
new_manifest = manifest_utils.rewrite_scps(new_manifest, puppet_account_id)
new_manifest = manifest_utils.parse_conditions(new_manifest)
if subset and subset.get("section"):
click.echo(f"Filtering for subset: {subset}")
new_manifest = manifest_utils.isolate(new_manifest, subset)
manifest_accounts_all = [
{"account_id": a.get("account_id"), "email": a.get("email")}
for a in new_manifest.get("accounts", [])
]
manifest_accounts_excluding = [
a for a in manifest_accounts_all if a.get("account_id") != puppet_account_id
]
# handle all accounts
sct_manifest_accounts = json.dumps(manifest_accounts_all)
sct_manifest_spokes = json.dumps(manifest_accounts_excluding)
regions = config.get_regions(puppet_account_id)
sct_config_regions = json.dumps(regions)
new_manifest["parameters"]["SCTManifestAccounts"] = dict(
default=sct_manifest_accounts
)
new_manifest["parameters"]["SCTManifestSpokes"] = dict(default=sct_manifest_spokes)
new_manifest["parameters"]["SCTConfigRegions"] = dict(default=sct_config_regions)
new_manifest["parameters"]["SCTAccountId"] = dict(default=puppet_account_id)
if new_manifest.get(constants.LAMBDA_INVOCATIONS) is None:
new_manifest[constants.LAMBDA_INVOCATIONS] = dict()
home_region = config.get_home_region(puppet_account_id)
with betterboto_client.ClientContextManager("ssm") as ssm:
response = ssm.get_parameter(Name="service-catalog-puppet-version")
version = response.get("Parameter").get("Value")
new_manifest["config_cache"] = dict(
home_region=home_region,
regions=regions,
should_collect_cloudformation_events=config.get_should_use_sns(
puppet_account_id, home_region
),
should_forward_events_to_eventbridge=config.get_should_use_eventbridge(
puppet_account_id, home_region
),
should_forward_failures_to_opscenter=config.get_should_forward_failures_to_opscenter(
puppet_account_id, home_region
),
puppet_version=version,
)
new_name = f.name.replace(".yaml", "-expanded.yaml")
logger.info("Writing new manifest: {}".format(new_name))
with open(new_name, "w") as output:
output.write(yaml_utils.dump(new_manifest))
def list_launches(f, format):
manifest = manifest_utils.load(f)
if format == "table":
click.echo("Getting details from your account...")
ALL_REGIONS = cli_command_helpers.get_regions(
os.environ.get("AWS_DEFAULT_REGION"))
deployment_map = manifest_utils.build_deployment_map(
manifest, constants.LAUNCHES)
account_ids = [a.get('account_id') for a in manifest.get('accounts')]
deployments = {}
for account_id in account_ids:
for region_name in ALL_REGIONS:
role = "arn:aws:iam::{}:role/{}".format(
account_id, 'servicecatalog-puppet/PuppetRole')
logger.info("Looking at region: {} in account: {}".format(
region_name, account_id))
with betterboto_client.CrossAccountClientContextManager(
'servicecatalog',
role,
'sc-{}-{}'.format(account_id, region_name),
region_name=region_name) as spoke_service_catalog:
response = spoke_service_catalog.list_accepted_portfolio_shares(
)
portfolios = response.get('PortfolioDetails', [])
response = spoke_service_catalog.list_portfolios()
portfolios += response.get('PortfolioDetails', [])
for portfolio in portfolios:
portfolio_id = portfolio.get('Id')
response = spoke_service_catalog.search_products_as_admin(
PortfolioId=portfolio_id)
for product_view_detail in response.get(
'ProductViewDetails', []):
product_view_summary = product_view_detail.get(
'ProductViewSummary')
product_id = product_view_summary.get('ProductId')
response = spoke_service_catalog.search_provisioned_products(
Filters={
'SearchQuery':
["productId:{}".format(product_id)]
})
for provisioned_product in response.get(
'ProvisionedProducts', []):
launch_name = provisioned_product.get('Name')
status = provisioned_product.get('Status')
provisioning_artifact_response = spoke_service_catalog.describe_provisioning_artifact(
ProvisioningArtifactId=provisioned_product.get(
'ProvisioningArtifactId'),
ProductId=provisioned_product.get('ProductId'),
).get('ProvisioningArtifactDetail')
if deployments.get(account_id) is None:
deployments[account_id] = {
'account_id': account_id,
constants.LAUNCHES: {}
}
if deployments[account_id][constants.LAUNCHES].get(
launch_name) is None:
deployments[account_id][
constants.LAUNCHES][launch_name] = {}
deployments[account_id][constants.LAUNCHES][
launch_name][region_name] = {
'launch_name':
launch_name,
'portfolio':
portfolio.get('DisplayName'),
'product':
manifest.get(constants.LAUNCHES,
{}).get(launch_name,
{}).get('product'),
'version':
provisioning_artifact_response.get('Name'),
'active':
provisioning_artifact_response.get(
'Active'),
'region':
region_name,
'status':
status,
}
output_path = os.path.sep.join([
constants.LAUNCHES_PATH,
account_id,
region_name,
])
if not os.path.exists(output_path):
os.makedirs(output_path)
output = os.path.sep.join([
output_path,
"{}.json".format(provisioned_product.get('Id'))
])
with open(output, 'w') as f:
f.write(
json.dumps(provisioned_product,
indent=4,
default=str))
results = {}
for account_id, details in deployment_map.items():
for launch_name, launch in details.get(constants.LAUNCHES, {}).items():
if deployments.get(account_id, {}).get(
constants.LAUNCHES, {}).get(launch_name) is None:
pass
else:
for region, regional_details in deployments[account_id][
constants.LAUNCHES][launch_name].items():
results[f"{account_id}_{region}_{launch_name}"] = {
'account_id': account_id,
'region': region,
'launch': launch_name,
'portfolio': regional_details.get('portfolio'),
'product': regional_details.get('product'),
'expected_version': launch.get('version'),
'actual_version': regional_details.get('version'),
'active': regional_details.get('active'),
'status': regional_details.get('status'),
}
if format == "table":
table = [[
'account_id',
'region',
'launch',
'portfolio',
'product',
'expected_version',
'actual_version',
'active',
'status',
]]
for result in results.values():
table.append([
result.get('account_id'),
result.get('region'),
result.get('launch'),
result.get('portfolio'),
result.get('product'),
result.get('expected_version'),
Color("{green}" + result.get('actual_version') + "{/green}") if
result.get('actual_version') == result.get('expected_version')
else Color("{red}" + result.get('actual_version') + "{/red}"),
Color("{green}" + str(result.get('active')) +
"{/green}") if result.get('active') else
Color("{red}" + str(result.get('active')) + "{/red}"),
Color("{green}" + result.get('status') +
"{/green}") if result.get('status') == "AVAILABLE" else
Color("{red}" + result.get('status') + "{/red}")
])
click.echo(terminaltables.AsciiTable(table).table)
elif format == "json":
click.echo(json.dumps(
results,
indent=4,
default=str,
))
else:
raise Exception(f"Unsupported format: {format}")
def generate_tasks(f, single_account=None, dry_run=False):
puppet_account_id = config.get_puppet_account_id()
manifest = manifest_utils.load(f)
tasks_to_run = []
should_use_sns = config.get_should_use_sns(
os.environ.get("AWS_DEFAULT_REGION"))
should_use_product_plans = config.get_should_use_product_plans(
os.environ.get("AWS_DEFAULT_REGION"))
task_defs = manifest_utils.convert_manifest_into_task_defs_for_launches(
manifest, puppet_account_id, should_use_sns, should_use_product_plans)
for task in task_defs:
if single_account is not None:
if task.get('account_id') != single_account:
continue
task_status = task.get('status')
del task['status']
if task_status == constants.PROVISIONED:
task['should_use_sns'] = should_use_sns
if dry_run:
tasks_to_run.append(
provisioning_tasks.ProvisionProductDryRunTask(**task))
else:
tasks_to_run.append(
provisioning_tasks.ProvisionProductTask(**task))
elif task_status == constants.TERMINATED:
for attribute in constants.DISALLOWED_ATTRIBUTES_FOR_TERMINATED_LAUNCHES:
logger.info(
f"checking {task.get('launch_name')} for disallowed attributes"
)
attribute_value = task.get(attribute)
if attribute_value is not None:
if isinstance(attribute_value, list):
if len(attribute_value) != 0:
raise Exception(
f"Launch {task.get('launch_name')} has disallowed attribute: {attribute}"
)
elif isinstance(attribute_value, dict):
if len(attribute_value.keys()) != 0:
raise Exception(
f"Launch {task.get('launch_name')} has disallowed attribute: {attribute}"
)
else:
raise Exception(
f"Launch {task.get('launch_name')} has disallowed attribute: {attribute}"
)
del task['launch_parameters']
del task['manifest_parameters']
del task['account_parameters']
del task['should_use_sns']
del task['requested_priority']
del task['should_use_product_plans']
del task['pre_actions']
del task['post_actions']
if dry_run:
tasks_to_run.append(
provisioning_tasks.TerminateProductDryRunTask(**task))
else:
tasks_to_run.append(
provisioning_tasks.TerminateProductTask(**task))
else:
raise Exception(f"Unsupported status of {task_status}")
if not dry_run:
spoke_local_portfolios_tasks = manifest_utils.convert_manifest_into_task_defs_for_spoke_local_portfolios(
manifest, puppet_account_id, should_use_sns, tasks_to_run)
for spoke_local_portfolios_task in spoke_local_portfolios_tasks:
if single_account is not None:
param_kwargs = spoke_local_portfolios_task.param_kwargs
logger.info(f"EPF:: {param_kwargs}")
if param_kwargs.get('account_id',
'not_an_account_id') != single_account:
continue
tasks_to_run.append(spoke_local_portfolios_task)
return tasks_to_run
def run(self):
self.info("started")
with open(self.manifest_file_path, "r") as m:
manifest = manifest_utils.load(m, self.puppet_account_id)
self.write_output(manifest)
self.info("Finished")
def generate_shares(f):
logger.info('Starting to generate shares for: {}'.format(f.name))
tasks_to_run = []
puppet_account_id = config.get_puppet_account_id()
manifest = manifest_utils.load(f)
accounts_by_id = {}
for account in manifest.get('accounts'):
accounts_by_id[account.get('account_id')] = account
launch_tasks = manifest_utils_for_launches.generate_launch_tasks(
manifest, puppet_account_id, False, False, include_expanded_from=False, single_account=None, is_dry_run=False
)
for launch_task in launch_tasks:
t = manifest_utils_for_launches.generate_launch_task_defs_for_launch(
launch_task.launch_name,
launch_task.manifest,
launch_task.puppet_account_id,
launch_task.should_use_sns,
launch_task.should_use_product_plans,
launch_task.include_expanded_from,
launch_task.single_account,
launch_task.is_dry_run
)
tasks = launch_task.generate_provisions(t.get('task_defs'))
for task_ in tasks:
task = task_.param_kwargs
a_id = task.get('account_id')
tasks_to_run.append(
portfoliomanagement_tasks.CreateShareForAccountLaunchRegion(
puppet_account_id=puppet_account_id,
account_id=a_id,
region=task.get('region'),
portfolio=task.get('portfolio'),
expanded_from=accounts_by_id.get(a_id).get('expanded_from'),
organization=accounts_by_id.get(a_id).get('organization'),
)
)
spoke_local_portfolios_tasks = manifest_utils_for_spoke_local_portfolios.generate_spoke_local_portfolios_tasks(
manifest, puppet_account_id, False, False, include_expanded_from=False, single_account=None, is_dry_run=False
)
# TODO fixme
for spoke_local_portfolios_task in spoke_local_portfolios_tasks:
t = manifest_utils_for_spoke_local_portfolios.generate_spoke_local_portfolios_tasks_for_spoke_local_portfolio(
spoke_local_portfolios_task.spoke_local_portfolio_name,
spoke_local_portfolios_task.manifest,
spoke_local_portfolios_task.puppet_account_id,
spoke_local_portfolios_task.should_use_sns,
spoke_local_portfolios_task.should_use_product_plans,
spoke_local_portfolios_task.include_expanded_from,
spoke_local_portfolios_task.single_account,
spoke_local_portfolios_task.is_dry_run,
)
sub_tasks = spoke_local_portfolios_task.generate_tasks(t.get('task_defs'))
for sub_task in sub_tasks:
if isinstance(sub_task, portfoliomanagement_tasks.CreateSpokeLocalPortfolioTask):
param_kwargs = sub_task.param_kwargs
a_id = param_kwargs.get('account_id')
tasks_to_run.append(
portfoliomanagement_tasks.CreateShareForAccountLaunchRegion(
puppet_account_id=puppet_account_id,
account_id=param_kwargs.get('account_id'),
region=param_kwargs.get('region'),
portfolio=param_kwargs.get('portfolio'),
expanded_from=accounts_by_id.get(a_id).get('expanded_from'),
organization=accounts_by_id.get(a_id).get('organization'),
)
)
runner.run_tasks_for_generate_shares(tasks_to_run)
def expand(f, puppet_account_id, single_account, subset=None):
click.echo("Expanding")
manifest = manifest_utils.load(f, puppet_account_id)
org_iam_role_arn = config.get_org_iam_role_arn(puppet_account_id)
if org_iam_role_arn is None:
click.echo("No org role set - not expanding")
new_manifest = manifest
else:
click.echo("Expanding using role: {}".format(org_iam_role_arn))
with betterboto_client.CrossAccountClientContextManager(
"organizations", org_iam_role_arn, "org-iam-role"
) as client:
new_manifest = manifest_utils.expand_manifest(manifest, client)
click.echo("Expanded")
if single_account:
click.echo(f"Filtering for single account: {single_account}")
for account in new_manifest.get("accounts", []):
if str(account.get("account_id")) == str(single_account):
click.echo(f"Found single account: {single_account}")
new_manifest["accounts"] = [account]
break
click.echo("Filtered")
new_manifest = manifest_utils.rewrite_depends_on(new_manifest)
new_manifest = manifest_utils.rewrite_ssm_parameters(new_manifest)
new_manifest = manifest_utils.rewrite_stacks(new_manifest, puppet_account_id)
if subset:
click.echo(f"Filtering for subset: {subset}")
new_manifest = manifest_utils.isolate(
manifest_utils.Manifest(new_manifest), subset
)
new_manifest = json.loads(json.dumps(new_manifest))
if new_manifest.get(constants.LAMBDA_INVOCATIONS) is None:
new_manifest[constants.LAMBDA_INVOCATIONS] = dict()
home_region = config.get_home_region(puppet_account_id)
with betterboto_client.ClientContextManager("ssm") as ssm:
response = ssm.get_parameter(Name="service-catalog-puppet-version")
version = response.get("Parameter").get("Value")
new_manifest["config_cache"] = dict(
home_region=home_region,
regions=config.get_regions(puppet_account_id, home_region),
should_collect_cloudformation_events=config.get_should_use_sns(
puppet_account_id, home_region
),
should_forward_events_to_eventbridge=config.get_should_use_eventbridge(
puppet_account_id, home_region
),
should_forward_failures_to_opscenter=config.get_should_forward_failures_to_opscenter(
puppet_account_id, home_region
),
puppet_version=version,
)
new_name = f.name.replace(".yaml", "-expanded.yaml")
logger.info("Writing new manifest: {}".format(new_name))
with open(new_name, "w") as output:
output.write(yaml.safe_dump(new_manifest, default_flow_style=False))
