def reset_provisioned_product_owner(f): puppet_account_id = config.get_puppet_account_id() current_account_id = puppet_account_id manifest = manifest_utils.load(f, puppet_account_id) os.environ["SCT_CACHE_INVALIDATOR"] = str(datetime.now()) task_defs = manifest_utils_for_launches.generate_launch_tasks( manifest, puppet_account_id, False, False) tasks_to_run = [] for task in task_defs: task_status = task.get("status") if task_status == constants.PROVISIONED: tasks_to_run.append( launch_tasks.ResetProvisionedProductOwnerTask( launch_name=task.get("launch_name"), account_id=task.get("account_id"), region=task.get("region"), )) runner.run_tasks( puppet_account_id, current_account_id, tasks_to_run, 10, execution_mode="hub", on_complete_url=None, )
def dry_run(f): manifest = manifest_utils.load(f) launch_tasks = {} tasks_to_run = [] all_launch_tasks = cli_command_helpers.deploy_launches(manifest) launch_tasks.update(all_launch_tasks) for task in cli_command_helpers.wire_dependencies(launch_tasks): task_status = task.get('status') del task['status'] if task_status == constants.PROVISIONED: tasks_to_run.append( luigi_tasks_and_targets.ProvisionProductDryRunTask(**task)) elif task_status == constants.TERMINATED: tasks_to_run.append( luigi_tasks_and_targets.TerminateProductDryRunTask(**task)) else: raise Exception(f"Unsupported status of {task_status}") # spoke_local_portfolio_tasks_to_run = cli_command_helpers.deploy_spoke_local_portfolios(manifest, launch_tasks) # tasks_to_run += spoke_local_portfolio_tasks_to_run cli_command_helpers.run_tasks_for_dry_run(tasks_to_run)
def reset_provisioned_product_owner(f): puppet_account_id = config.get_puppet_account_id() current_account_id = puppet_account_id manifest = manifest_utils.load(f, puppet_account_id) task_defs = manifest_utils_for_launches.generate_launch_tasks( manifest, puppet_account_id, False, False ) tasks_to_run = [] for task in task_defs: task_status = task.get("status") if task_status == constants.PROVISIONED: tasks_to_run.append( provisioning_tasks.ResetProvisionedProductOwnerTask( launch_name=task.get("launch_name"), account_id=task.get("account_id"), region=task.get("region"), ) ) cache_invalidator = str(datetime.now()) runner.run_tasks( puppet_account_id, current_account_id, tasks_to_run, 10, cache_invalidator=cache_invalidator, on_complete_url=None, )
def expand(f, single_account): click.echo("Expanding") puppet_account_id = config.get_puppet_account_id() manifest = manifest_utils.load(f, puppet_account_id) org_iam_role_arn = config.get_org_iam_role_arn(puppet_account_id) if org_iam_role_arn is None: click.echo("No org role set - not expanding") new_manifest = manifest else: click.echo("Expanding using role: {}".format(org_iam_role_arn)) with betterboto_client.CrossAccountClientContextManager( "organizations", org_iam_role_arn, "org-iam-role" ) as client: new_manifest = manifest_utils.expand_manifest(manifest, client) click.echo("Expanded") if single_account: click.echo(f"Filtering for single account: {single_account}") for account in new_manifest.get("accounts", []): if account.get("account_id") == single_account: click.echo(f"Found single account: {single_account}") new_manifest["accounts"] = [account] break click.echo("Filtered") new_name = f.name.replace(".yaml", "-expanded.yaml") logger.info("Writing new manifest: {}".format(new_name)) with open(new_name, "w") as output: output.write(yaml.safe_dump(new_manifest, default_flow_style=False))
def generate_tasks(f, single_account=None, is_dry_run=False): puppet_account_id = config.get_puppet_account_id() manifest = manifest_utils.load(f) should_use_sns = config.get_should_use_sns(os.environ.get("AWS_DEFAULT_REGION")) should_use_product_plans = config.get_should_use_product_plans(os.environ.get("AWS_DEFAULT_REGION")) tasks_to_run = manifest_utils_for_launches.generate_launch_tasks( manifest, puppet_account_id, should_use_sns, should_use_product_plans, include_expanded_from=False, single_account=single_account, is_dry_run=is_dry_run, ) logger.info("Finished generating provisioning tasks") if not is_dry_run: logger.info("Generating sharing tasks") spoke_local_portfolios_tasks = manifest_utils_for_spoke_local_portfolios.generate_spoke_local_portfolios_tasks( manifest, puppet_account_id, should_use_sns, should_use_product_plans, include_expanded_from=False, single_account=single_account, is_dry_run=is_dry_run, ) tasks_to_run += spoke_local_portfolios_tasks logger.info("Finished generating sharing tasks") logger.info("Finished generating all tasks") return tasks_to_run
def deploy(f, single_account): puppet_account_id = cli_command_helpers.get_puppet_account_id() manifest = manifest_utils.load(f) launch_tasks = {} tasks_to_run = [] should_use_sns = cli_command_helpers.get_should_use_sns( os.environ.get("AWS_DEFAULT_REGION")) all_launch_tasks = cli_command_helpers.deploy_launches( manifest, puppet_account_id) launch_tasks.update(all_launch_tasks) for task in cli_command_helpers.wire_dependencies(launch_tasks): task_status = task.get('status') del task['status'] if task_status == constants.PROVISIONED: task['should_use_sns'] = should_use_sns tasks_to_run.append( luigi_tasks_and_targets.ProvisionProductTask(**task)) elif task_status == constants.TERMINATED: for attribute in constants.DISALLOWED_ATTRIBUTES_FOR_TERMINATED_LAUNCHES: logger.info( f"checking {task.get('launch_name')} for disallowed attributes" ) attribute_value = task.get(attribute) if attribute_value is not None: if isinstance(attribute_value, list): if len(attribute_value) != 0: raise Exception( f"Launch {task.get('launch_name')} has disallowed attribute: {attribute}" ) elif isinstance(attribute_value, dict): if len(attribute_value.keys()) != 0: raise Exception( f"Launch {task.get('launch_name')} has disallowed attribute: {attribute}" ) else: raise Exception( f"Launch {task.get('launch_name')} has disallowed attribute: {attribute}" ) tasks_to_run.append( luigi_tasks_and_targets.TerminateProductTask(**task)) else: raise Exception(f"Unsupported status of {task_status}") spoke_local_portfolio_tasks_to_run = cli_command_helpers.deploy_spoke_local_portfolios( manifest, launch_tasks, should_use_sns, puppet_account_id) tasks_to_run += spoke_local_portfolio_tasks_to_run cli_command_helpers.run_tasks(tasks_to_run)
def generate_shares(f): logger.info('Starting to generate shares for: {}'.format(f.name)) manifest = manifest_utils.load(f) deployment_map = manifest_utils.build_deployment_map( manifest, constants.LAUNCHES) import_map = manifest_utils.build_deployment_map( manifest, constants.SPOKE_LOCAL_PORTFOLIOS) cli_command_helpers.create_share_template( deployment_map, import_map, cli_command_helpers.get_puppet_account_id())
def expand(f): click.echo('Expanding') manifest = manifest_utils.load(f) org_iam_role_arn = cli_command_helpers.get_org_iam_role_arn() if org_iam_role_arn is None: click.echo('No org role set - not expanding') new_manifest = manifest else: click.echo('Expanding using role: {}'.format(org_iam_role_arn)) with betterboto_client.CrossAccountClientContextManager( 'organizations', org_iam_role_arn, 'org-iam-role') as client: new_manifest = manifest_utils.expand_manifest(manifest, client) click.echo('Expanded') new_name = f.name.replace(".yaml", '-expanded.yaml') logger.info('Writing new manifest: {}'.format(new_name)) with open(new_name, 'w') as output: output.write(yaml.safe_dump(new_manifest, default_flow_style=False))
def explode(f): logger.info("Exploding") puppet_account_id = config.get_puppet_account_id() original_name = f.name expanded_output = f.name.replace(".yaml", "-expanded.yaml") expanded_manifest = manifest_utils.load( open(expanded_output, "r"), puppet_account_id ) expanded_manifest = manifest_utils.Manifest(expanded_manifest) exploded = manifest_utils.explode(expanded_manifest) logger.info(f"found {len(exploded)} graphs") count = 0 for mani in exploded: with open(original_name.replace(".yaml", f"-exploded-{count}.yaml"), "w") as f: f.write(yaml.safe_dump(json.loads(json.dumps(mani)))) count += 1
def validate(f): logger.info("Validating {}".format(f.name)) manifest = manifest_utils.load(f, config.get_puppet_account_id()) schema = yamale.make_schema( asset_helpers.resolve_from_site_packages("schema.yaml")) data = yamale.make_data(content=yaml.safe_dump(manifest)) yamale.validate(schema, data, strict=False) tags_defined_by_accounts = {} for account in manifest.get("accounts"): for tag in account.get("tags", []): tags_defined_by_accounts[tag] = True for collection_type in constants.ALL_SECTION_NAMES: collection_to_check = manifest.get(collection_type, {}) for collection_name, collection_item in collection_to_check.items(): for deploy_to in collection_item.get("deploy_to", {}).get("tags", []): tag_to_check = deploy_to.get("tag") if tags_defined_by_accounts.get(tag_to_check) is None: print( f"{collection_type}.{collection_name} uses tag {tag_to_check} in deploy_to that does not exist" ) for depends_on in collection_item.get("depends_on", []): if isinstance(depends_on, str): if manifest.get( constants.LAUNCHES).get(depends_on) is None: print( f"{collection_type}.{collection_name} uses {depends_on} in depends_on that does not exist" ) else: tt = constants.SECTION_SINGULAR_TO_PLURAL.get( depends_on.get("type", constants.LAUNCH)) dd = depends_on.get("name") if manifest.get(tt).get(dd) is None: print( f"{collection_type}.{collection_name} uses {depends_on} in depends_on that does not exist" ) click.echo("Finished validating: {}".format(f.name)) click.echo("Finished validating: OK")
def reset_provisioned_product_owner(f): manifest = manifest_utils.load(f) launch_tasks = {} tasks_to_run = [] all_launch_tasks = cli_command_helpers.deploy_launches(manifest) launch_tasks.update(all_launch_tasks) for task in cli_command_helpers.wire_dependencies(launch_tasks): task_status = task.get('status') del task['status'] if task_status == constants.PROVISIONED: tasks_to_run.append( luigi_tasks_and_targets.ResetProvisionedProductOwnerTask( **task)) cli_command_helpers.run_tasks(tasks_to_run)
def reset_provisioned_product_owner(f): puppet_account_id = config.get_puppet_account_id() manifest = manifest_utils.load(f) task_defs = manifest_utils.convert_manifest_into_task_defs_for_launches( manifest, puppet_account_id, False, False) tasks_to_run = [] for task in task_defs: task_status = task.get('status') if task_status == constants.PROVISIONED: tasks_to_run.append( provisioning_tasks.ResetProvisionedProductOwnerTask( launch_name=task.get('launch_name'), account_id=task.get('account_id'), region=task.get('region'), )) runner.run_tasks(tasks_to_run, 10)
def expand(f, single_account, subset=None): click.echo("Expanding") puppet_account_id = config.get_puppet_account_id() manifest = manifest_utils.load(f, puppet_account_id) org_iam_role_arn = config.get_org_iam_role_arn(puppet_account_id) if org_iam_role_arn is None: click.echo("No org role set - not expanding") new_manifest = manifest else: click.echo("Expanding using role: {}".format(org_iam_role_arn)) with betterboto_client.CrossAccountClientContextManager( "organizations", org_iam_role_arn, "org-iam-role") as client: new_manifest = manifest_utils.expand_manifest(manifest, client) click.echo("Expanded") if single_account: click.echo(f"Filtering for single account: {single_account}") for account in new_manifest.get("accounts", []): if str(account.get("account_id")) == str(single_account): click.echo(f"Found single account: {single_account}") new_manifest["accounts"] = [account] break click.echo("Filtered") new_manifest = manifest_utils.rewrite_depends_on(new_manifest) if subset: click.echo(f"Filtering for subset: {subset}") new_manifest = manifest_utils.isolate( manifest_utils.Manifest(new_manifest), subset) new_manifest = json.loads(json.dumps(new_manifest)) if new_manifest.get(constants.LAMBDA_INVOCATIONS) is None: new_manifest[constants.LAMBDA_INVOCATIONS] = dict() new_name = f.name.replace(".yaml", "-expanded.yaml") logger.info("Writing new manifest: {}".format(new_name)) with open(new_name, "w") as output: output.write(yaml.safe_dump(new_manifest, default_flow_style=False))
def generate_shares(f): logger.info('Starting to generate shares for: {}'.format(f.name)) tasks_to_run = [] puppet_account_id = config.get_puppet_account_id() manifest = manifest_utils.load(f) task_defs = manifest_utils.convert_manifest_into_task_defs_for_launches( manifest, puppet_account_id, False, False, include_expanded_from=True) for task in task_defs: tasks_to_run.append( portfoliomanagement_tasks.CreateShareForAccountLaunchRegion( puppet_account_id=puppet_account_id, account_id=task.get('account_id'), region=task.get('region'), portfolio=task.get('portfolio'), expanded_from=task.get('expanded_from'), organization=task.get('organization'), )) spoke_local_portfolios_tasks = manifest_utils.convert_manifest_into_task_defs_for_spoke_local_portfolios( manifest, puppet_account_id, False, tasks_to_run) for task in spoke_local_portfolios_tasks: if isinstance(task, portfoliomanagement_tasks.CreateSpokeLocalPortfolioTask): param_kwargs = task.param_kwargs tasks_to_run.append( portfoliomanagement_tasks.CreateShareForAccountLaunchRegion( puppet_account_id=puppet_account_id, account_id=param_kwargs.get('account_id'), region=param_kwargs.get('region'), portfolio=param_kwargs.get('portfolio'), expanded_from=param_kwargs.get('expanded_from'), organization=param_kwargs.get('organization'), )) runner.run_tasks_for_generate_shares(tasks_to_run)
def validate(f): logger.info("Validating {}".format(f.name)) manifest = manifest_utils.load(f, config.get_puppet_account_id()) try: Loader = yaml.CSafeLoader except AttributeError: # System does not have libyaml Loader = yaml.SafeLoader Loader.add_constructor("!Equals", yaml_utils.Equals.from_yaml) Loader.add_constructor("!Not", yaml_utils.Not.from_yaml) schema = yamale.make_schema(asset_helpers.resolve_from_site_packages("schema.yaml")) data = yamale.make_data(content=yaml_utils.dump(manifest)) yamale.validate(schema, data, strict=False) has_default_default_region = ( manifest.get("defaults", {}).get("accounts", {}).get("default_region", False) ) has_default_regions_enabled = ( manifest.get("defaults", {}).get("accounts", {}).get("regions_enabled", False) ) tags_defined_by_accounts = {} for account in manifest.get("accounts"): account_entry_is_an_overwrite_or_append = account.get( "append", account.get("overwrite", False) ) if ( not account_entry_is_an_overwrite_or_append and account.get("default_region") is None and not has_default_default_region ): raise Exception( f"account entry {account.get('account_id', account.get('ou'))} is missing default_region" ) if ( not account_entry_is_an_overwrite_or_append and account.get("regions_enabled") is None and not has_default_regions_enabled ): raise Exception( f"account entry {account.get('account_id', account.get('ou'))} is missing regions_enabled" ) for tag in account.get("tags", []): tags_defined_by_accounts[tag] = True for collection_type in constants.ALL_SECTION_NAMES: collection_to_check = manifest.get(collection_type, {}) for collection_name, collection_item in collection_to_check.items(): for deploy_to in collection_item.get("deploy_to", {}).get("tags", []): tag_to_check = deploy_to.get("tag") if tags_defined_by_accounts.get(tag_to_check) is None: print( f"{collection_type}.{collection_name} uses tag {tag_to_check} in deploy_to that does not exist" ) for depends_on in collection_item.get("depends_on", []): if isinstance(depends_on, str): if manifest.get(constants.LAUNCHES).get(depends_on) is None: print( f"{collection_type}.{collection_name} uses {depends_on} in depends_on that does not exist" ) else: tt = constants.SECTION_SINGULAR_TO_PLURAL.get( depends_on.get("type", constants.LAUNCH) ) dd = depends_on.get("name") if manifest.get(tt).get(dd) is None: print( f"{collection_type}.{collection_name} uses {depends_on} in depends_on that does not exist" ) click.echo("Finished validating: {}".format(f.name)) click.echo("Finished validating: OK")
def expand(f, puppet_account_id, single_account, subset=None): click.echo("Expanding") target_directory = os.path.sep.join([os.path.dirname(f.name), "manifests"]) assemble_manifest_from_ssm(target_directory) manifest = manifest_utils.load(f, puppet_account_id) org_iam_role_arn = config.get_org_iam_role_arn(puppet_account_id) if org_iam_role_arn is None: click.echo("No org role set - not expanding") new_manifest = manifest else: click.echo("Expanding using role: {}".format(org_iam_role_arn)) with betterboto_client.CrossAccountClientContextManager( "organizations", org_iam_role_arn, "org-iam-role" ) as client: new_manifest = manifest_utils.expand_manifest(manifest, client) click.echo("Expanded") new_manifest = manifest_utils.rewrite_deploy_as_share_to_for_spoke_local_portfolios( new_manifest ) if single_account: click.echo(f"Filtering for single account: {single_account}") for account in new_manifest.get("accounts", []): if str(account.get("account_id")) == str(single_account): click.echo(f"Found single account: {single_account}") new_manifest["accounts"] = [account] break items_to_delete = list() for section_name in constants.ALL_SECTION_NAMES: deploy_to_name = constants.DEPLOY_TO_NAMES[section_name] for item_name, item in new_manifest.get(section_name, {}).items(): accounts = list() for deploy_details in item.get(deploy_to_name, {}).get("accounts", []): if str(deploy_details.get("account_id")) == str(single_account): accounts.append(deploy_details) print(f"{item_name}: there are " + str(len(accounts))) if item.get(deploy_to_name).get("accounts"): if len(accounts) > 0: item[deploy_to_name]["accounts"] = accounts else: if item[deploy_to_name].get("tags") or item[deploy_to_name].get( "ous" ): del item[deploy_to_name]["accounts"] else: items_to_delete.append(f"{section_name}:{item_name}") for item_to_delete in items_to_delete: section_name, item_name = item_to_delete.split(":") del new_manifest[section_name][item_name] click.echo("Filtered") new_manifest = manifest_utils.rewrite_cfct(new_manifest) new_manifest = manifest_utils.rewrite_depends_on(new_manifest) new_manifest = manifest_utils.rewrite_ssm_parameters(new_manifest) new_manifest = manifest_utils.rewrite_stacks(new_manifest, puppet_account_id) new_manifest = manifest_utils.rewrite_scps(new_manifest, puppet_account_id) new_manifest = manifest_utils.parse_conditions(new_manifest) if subset and subset.get("section"): click.echo(f"Filtering for subset: {subset}") new_manifest = manifest_utils.isolate(new_manifest, subset) manifest_accounts_all = [ {"account_id": a.get("account_id"), "email": a.get("email")} for a in new_manifest.get("accounts", []) ] manifest_accounts_excluding = [ a for a in manifest_accounts_all if a.get("account_id") != puppet_account_id ] # handle all accounts sct_manifest_accounts = json.dumps(manifest_accounts_all) sct_manifest_spokes = json.dumps(manifest_accounts_excluding) regions = config.get_regions(puppet_account_id) sct_config_regions = json.dumps(regions) new_manifest["parameters"]["SCTManifestAccounts"] = dict( default=sct_manifest_accounts ) new_manifest["parameters"]["SCTManifestSpokes"] = dict(default=sct_manifest_spokes) new_manifest["parameters"]["SCTConfigRegions"] = dict(default=sct_config_regions) new_manifest["parameters"]["SCTAccountId"] = dict(default=puppet_account_id) if new_manifest.get(constants.LAMBDA_INVOCATIONS) is None: new_manifest[constants.LAMBDA_INVOCATIONS] = dict() home_region = config.get_home_region(puppet_account_id) with betterboto_client.ClientContextManager("ssm") as ssm: response = ssm.get_parameter(Name="service-catalog-puppet-version") version = response.get("Parameter").get("Value") new_manifest["config_cache"] = dict( home_region=home_region, regions=regions, should_collect_cloudformation_events=config.get_should_use_sns( puppet_account_id, home_region ), should_forward_events_to_eventbridge=config.get_should_use_eventbridge( puppet_account_id, home_region ), should_forward_failures_to_opscenter=config.get_should_forward_failures_to_opscenter( puppet_account_id, home_region ), puppet_version=version, ) new_name = f.name.replace(".yaml", "-expanded.yaml") logger.info("Writing new manifest: {}".format(new_name)) with open(new_name, "w") as output: output.write(yaml_utils.dump(new_manifest))
def list_launches(f, format): manifest = manifest_utils.load(f) if format == "table": click.echo("Getting details from your account...") ALL_REGIONS = cli_command_helpers.get_regions( os.environ.get("AWS_DEFAULT_REGION")) deployment_map = manifest_utils.build_deployment_map( manifest, constants.LAUNCHES) account_ids = [a.get('account_id') for a in manifest.get('accounts')] deployments = {} for account_id in account_ids: for region_name in ALL_REGIONS: role = "arn:aws:iam::{}:role/{}".format( account_id, 'servicecatalog-puppet/PuppetRole') logger.info("Looking at region: {} in account: {}".format( region_name, account_id)) with betterboto_client.CrossAccountClientContextManager( 'servicecatalog', role, 'sc-{}-{}'.format(account_id, region_name), region_name=region_name) as spoke_service_catalog: response = spoke_service_catalog.list_accepted_portfolio_shares( ) portfolios = response.get('PortfolioDetails', []) response = spoke_service_catalog.list_portfolios() portfolios += response.get('PortfolioDetails', []) for portfolio in portfolios: portfolio_id = portfolio.get('Id') response = spoke_service_catalog.search_products_as_admin( PortfolioId=portfolio_id) for product_view_detail in response.get( 'ProductViewDetails', []): product_view_summary = product_view_detail.get( 'ProductViewSummary') product_id = product_view_summary.get('ProductId') response = spoke_service_catalog.search_provisioned_products( Filters={ 'SearchQuery': ["productId:{}".format(product_id)] }) for provisioned_product in response.get( 'ProvisionedProducts', []): launch_name = provisioned_product.get('Name') status = provisioned_product.get('Status') provisioning_artifact_response = spoke_service_catalog.describe_provisioning_artifact( ProvisioningArtifactId=provisioned_product.get( 'ProvisioningArtifactId'), ProductId=provisioned_product.get('ProductId'), ).get('ProvisioningArtifactDetail') if deployments.get(account_id) is None: deployments[account_id] = { 'account_id': account_id, constants.LAUNCHES: {} } if deployments[account_id][constants.LAUNCHES].get( launch_name) is None: deployments[account_id][ constants.LAUNCHES][launch_name] = {} deployments[account_id][constants.LAUNCHES][ launch_name][region_name] = { 'launch_name': launch_name, 'portfolio': portfolio.get('DisplayName'), 'product': manifest.get(constants.LAUNCHES, {}).get(launch_name, {}).get('product'), 'version': provisioning_artifact_response.get('Name'), 'active': provisioning_artifact_response.get( 'Active'), 'region': region_name, 'status': status, } output_path = os.path.sep.join([ constants.LAUNCHES_PATH, account_id, region_name, ]) if not os.path.exists(output_path): os.makedirs(output_path) output = os.path.sep.join([ output_path, "{}.json".format(provisioned_product.get('Id')) ]) with open(output, 'w') as f: f.write( json.dumps(provisioned_product, indent=4, default=str)) results = {} for account_id, details in deployment_map.items(): for launch_name, launch in details.get(constants.LAUNCHES, {}).items(): if deployments.get(account_id, {}).get( constants.LAUNCHES, {}).get(launch_name) is None: pass else: for region, regional_details in deployments[account_id][ constants.LAUNCHES][launch_name].items(): results[f"{account_id}_{region}_{launch_name}"] = { 'account_id': account_id, 'region': region, 'launch': launch_name, 'portfolio': regional_details.get('portfolio'), 'product': regional_details.get('product'), 'expected_version': launch.get('version'), 'actual_version': regional_details.get('version'), 'active': regional_details.get('active'), 'status': regional_details.get('status'), } if format == "table": table = [[ 'account_id', 'region', 'launch', 'portfolio', 'product', 'expected_version', 'actual_version', 'active', 'status', ]] for result in results.values(): table.append([ result.get('account_id'), result.get('region'), result.get('launch'), result.get('portfolio'), result.get('product'), result.get('expected_version'), Color("{green}" + result.get('actual_version') + "{/green}") if result.get('actual_version') == result.get('expected_version') else Color("{red}" + result.get('actual_version') + "{/red}"), Color("{green}" + str(result.get('active')) + "{/green}") if result.get('active') else Color("{red}" + str(result.get('active')) + "{/red}"), Color("{green}" + result.get('status') + "{/green}") if result.get('status') == "AVAILABLE" else Color("{red}" + result.get('status') + "{/red}") ]) click.echo(terminaltables.AsciiTable(table).table) elif format == "json": click.echo(json.dumps( results, indent=4, default=str, )) else: raise Exception(f"Unsupported format: {format}")
def generate_tasks(f, single_account=None, dry_run=False): puppet_account_id = config.get_puppet_account_id() manifest = manifest_utils.load(f) tasks_to_run = [] should_use_sns = config.get_should_use_sns( os.environ.get("AWS_DEFAULT_REGION")) should_use_product_plans = config.get_should_use_product_plans( os.environ.get("AWS_DEFAULT_REGION")) task_defs = manifest_utils.convert_manifest_into_task_defs_for_launches( manifest, puppet_account_id, should_use_sns, should_use_product_plans) for task in task_defs: if single_account is not None: if task.get('account_id') != single_account: continue task_status = task.get('status') del task['status'] if task_status == constants.PROVISIONED: task['should_use_sns'] = should_use_sns if dry_run: tasks_to_run.append( provisioning_tasks.ProvisionProductDryRunTask(**task)) else: tasks_to_run.append( provisioning_tasks.ProvisionProductTask(**task)) elif task_status == constants.TERMINATED: for attribute in constants.DISALLOWED_ATTRIBUTES_FOR_TERMINATED_LAUNCHES: logger.info( f"checking {task.get('launch_name')} for disallowed attributes" ) attribute_value = task.get(attribute) if attribute_value is not None: if isinstance(attribute_value, list): if len(attribute_value) != 0: raise Exception( f"Launch {task.get('launch_name')} has disallowed attribute: {attribute}" ) elif isinstance(attribute_value, dict): if len(attribute_value.keys()) != 0: raise Exception( f"Launch {task.get('launch_name')} has disallowed attribute: {attribute}" ) else: raise Exception( f"Launch {task.get('launch_name')} has disallowed attribute: {attribute}" ) del task['launch_parameters'] del task['manifest_parameters'] del task['account_parameters'] del task['should_use_sns'] del task['requested_priority'] del task['should_use_product_plans'] del task['pre_actions'] del task['post_actions'] if dry_run: tasks_to_run.append( provisioning_tasks.TerminateProductDryRunTask(**task)) else: tasks_to_run.append( provisioning_tasks.TerminateProductTask(**task)) else: raise Exception(f"Unsupported status of {task_status}") if not dry_run: spoke_local_portfolios_tasks = manifest_utils.convert_manifest_into_task_defs_for_spoke_local_portfolios( manifest, puppet_account_id, should_use_sns, tasks_to_run) for spoke_local_portfolios_task in spoke_local_portfolios_tasks: if single_account is not None: param_kwargs = spoke_local_portfolios_task.param_kwargs logger.info(f"EPF:: {param_kwargs}") if param_kwargs.get('account_id', 'not_an_account_id') != single_account: continue tasks_to_run.append(spoke_local_portfolios_task) return tasks_to_run
def run(self): self.info("started") with open(self.manifest_file_path, "r") as m: manifest = manifest_utils.load(m, self.puppet_account_id) self.write_output(manifest) self.info("Finished")
def generate_shares(f): logger.info('Starting to generate shares for: {}'.format(f.name)) tasks_to_run = [] puppet_account_id = config.get_puppet_account_id() manifest = manifest_utils.load(f) accounts_by_id = {} for account in manifest.get('accounts'): accounts_by_id[account.get('account_id')] = account launch_tasks = manifest_utils_for_launches.generate_launch_tasks( manifest, puppet_account_id, False, False, include_expanded_from=False, single_account=None, is_dry_run=False ) for launch_task in launch_tasks: t = manifest_utils_for_launches.generate_launch_task_defs_for_launch( launch_task.launch_name, launch_task.manifest, launch_task.puppet_account_id, launch_task.should_use_sns, launch_task.should_use_product_plans, launch_task.include_expanded_from, launch_task.single_account, launch_task.is_dry_run ) tasks = launch_task.generate_provisions(t.get('task_defs')) for task_ in tasks: task = task_.param_kwargs a_id = task.get('account_id') tasks_to_run.append( portfoliomanagement_tasks.CreateShareForAccountLaunchRegion( puppet_account_id=puppet_account_id, account_id=a_id, region=task.get('region'), portfolio=task.get('portfolio'), expanded_from=accounts_by_id.get(a_id).get('expanded_from'), organization=accounts_by_id.get(a_id).get('organization'), ) ) spoke_local_portfolios_tasks = manifest_utils_for_spoke_local_portfolios.generate_spoke_local_portfolios_tasks( manifest, puppet_account_id, False, False, include_expanded_from=False, single_account=None, is_dry_run=False ) # TODO fixme for spoke_local_portfolios_task in spoke_local_portfolios_tasks: t = manifest_utils_for_spoke_local_portfolios.generate_spoke_local_portfolios_tasks_for_spoke_local_portfolio( spoke_local_portfolios_task.spoke_local_portfolio_name, spoke_local_portfolios_task.manifest, spoke_local_portfolios_task.puppet_account_id, spoke_local_portfolios_task.should_use_sns, spoke_local_portfolios_task.should_use_product_plans, spoke_local_portfolios_task.include_expanded_from, spoke_local_portfolios_task.single_account, spoke_local_portfolios_task.is_dry_run, ) sub_tasks = spoke_local_portfolios_task.generate_tasks(t.get('task_defs')) for sub_task in sub_tasks: if isinstance(sub_task, portfoliomanagement_tasks.CreateSpokeLocalPortfolioTask): param_kwargs = sub_task.param_kwargs a_id = param_kwargs.get('account_id') tasks_to_run.append( portfoliomanagement_tasks.CreateShareForAccountLaunchRegion( puppet_account_id=puppet_account_id, account_id=param_kwargs.get('account_id'), region=param_kwargs.get('region'), portfolio=param_kwargs.get('portfolio'), expanded_from=accounts_by_id.get(a_id).get('expanded_from'), organization=accounts_by_id.get(a_id).get('organization'), ) ) runner.run_tasks_for_generate_shares(tasks_to_run)
def expand(f, puppet_account_id, single_account, subset=None): click.echo("Expanding") manifest = manifest_utils.load(f, puppet_account_id) org_iam_role_arn = config.get_org_iam_role_arn(puppet_account_id) if org_iam_role_arn is None: click.echo("No org role set - not expanding") new_manifest = manifest else: click.echo("Expanding using role: {}".format(org_iam_role_arn)) with betterboto_client.CrossAccountClientContextManager( "organizations", org_iam_role_arn, "org-iam-role" ) as client: new_manifest = manifest_utils.expand_manifest(manifest, client) click.echo("Expanded") if single_account: click.echo(f"Filtering for single account: {single_account}") for account in new_manifest.get("accounts", []): if str(account.get("account_id")) == str(single_account): click.echo(f"Found single account: {single_account}") new_manifest["accounts"] = [account] break click.echo("Filtered") new_manifest = manifest_utils.rewrite_depends_on(new_manifest) new_manifest = manifest_utils.rewrite_ssm_parameters(new_manifest) new_manifest = manifest_utils.rewrite_stacks(new_manifest, puppet_account_id) if subset: click.echo(f"Filtering for subset: {subset}") new_manifest = manifest_utils.isolate( manifest_utils.Manifest(new_manifest), subset ) new_manifest = json.loads(json.dumps(new_manifest)) if new_manifest.get(constants.LAMBDA_INVOCATIONS) is None: new_manifest[constants.LAMBDA_INVOCATIONS] = dict() home_region = config.get_home_region(puppet_account_id) with betterboto_client.ClientContextManager("ssm") as ssm: response = ssm.get_parameter(Name="service-catalog-puppet-version") version = response.get("Parameter").get("Value") new_manifest["config_cache"] = dict( home_region=home_region, regions=config.get_regions(puppet_account_id, home_region), should_collect_cloudformation_events=config.get_should_use_sns( puppet_account_id, home_region ), should_forward_events_to_eventbridge=config.get_should_use_eventbridge( puppet_account_id, home_region ), should_forward_failures_to_opscenter=config.get_should_forward_failures_to_opscenter( puppet_account_id, home_region ), puppet_version=version, ) new_name = f.name.replace(".yaml", "-expanded.yaml") logger.info("Writing new manifest: {}".format(new_name)) with open(new_name, "w") as output: output.write(yaml.safe_dump(new_manifest, default_flow_style=False))