def handle_action_execution_detail(puppet_account_id, action_execution_detail):
action_type_id = action_execution_detail.get("input").get("actionTypeId")
if (
action_type_id.get("category") == "Build"
and action_type_id.get("owner") == "AWS"
and action_type_id.get("provider") == "CodeBuild"
):
external_execution_id = (
action_execution_detail.get("output")
.get("executionResult")
.get("externalExecutionId")
)
with betterboto_client.ClientContextManager(
"codebuild", region_name=config.get_home_region(puppet_account_id)
) as codebuild:
builds = codebuild.batch_get_builds(ids=[external_execution_id]).get(
"builds"
)
build = builds[0]
log_details = build.get("logs")
with betterboto_client.ClientContextManager(
"logs", region_name=config.get_home_region(puppet_account_id)
) as logs:
with open(
f"log-{action_execution_detail.get('input').get('configuration').get('ProjectName')}.log",
"w",
) as f:
params = {
"logGroupName": log_details.get("groupName"),
"logStreamName": log_details.get("streamName"),
"startFromHead": True,
}
has_more_logs = True
while has_more_logs:
get_log_events_response = logs.get_log_events(**params)
if (len(get_log_events_response.get("events"))) > 0:
params["nextToken"] = get_log_events_response.get(
"nextForwardToken"
)
else:
has_more_logs = False
if params.get("nextToken"):
del params["nextToken"]
for e in get_log_events_response.get("events"):
d = datetime.utcfromtimestamp(
e.get("timestamp") / 1000
).strftime("%Y-%m-%d %H:%M:%S")
f.write(f"{d} : {e.get('message')}")
def run(self):
home_region = config.get_home_region(self.puppet_account_id)
with betterboto_client.CrossAccountClientContextManager(
"lambda",
f"arn:aws:iam::{self.puppet_account_id}:role/servicecatalog-puppet/PuppetRole",
f"sc-{home_region}-{self.puppet_account_id}",
region_name=home_region,
) as lambda_client:
payload = dict(
account_id=self.account_id,
region=self.region,
parameters=self.get_all_params(),
)
response = lambda_client.invoke(
FunctionName=self.function_name,
InvocationType=self.invocation_type,
Payload=json.dumps(payload),
Qualifier=self.qualifier,
)
success_results = dict(RequestResponse=200, Event=202, DryRun=204)
if success_results.get(self.invocation_type) != response.get('StatusCode'):
raise Exception(f"{self.lambda_invocation_name} failed for {self.account_id}, {self.region}")
else:
if response.get('FunctionError'):
error_payload = response.get('Payload').read()
raise Exception(error_payload)
else:
output = dict(**self.params_for_results_display(), payload=payload, response=response)
self.write_output(output)
def requires(self):
all_params = dict()
all_params.update(self.manifest_parameters)
all_params.update(self.launch_parameters)
all_params.update(self.account_parameters)
ssm_params = dict()
for param_name, param_details in all_params.items():
if param_details.get("ssm"):
if param_details.get("default"):
del param_details["default"]
ssm_params[param_name] = tasks.GetSSMParamTask(
parameter_name=param_name,
name=param_details.get("ssm").get("name"),
region=param_details.get("ssm").get(
"region",
config.get_home_region(self.puppet_account_id)),
cache_invalidator=self.cache_invalidator,
)
self.all_params = all_params
return dict(
ssm_params=ssm_params,
dependencies=LambdaInvocationDependenciesWrapperTask(
lambda_invocation_name=self.lambda_invocation_name,
manifest_file_path=self.manifest_file_path,
puppet_account_id=self.puppet_account_id,
should_use_sns=self.should_use_sns,
should_use_product_plans=self.should_use_product_plans,
include_expanded_from=self.include_expanded_from,
single_account=self.single_account,
is_dry_run=self.is_dry_run,
cache_invalidator=self.cache_invalidator,
),
)
def run(self):
home_region = config.get_home_region(self.puppet_account_id)
with self.hub_regional_client(
"lambda", region_name=home_region) as lambda_client:
payload = dict(
account_id=self.account_id,
region=self.region,
parameters=self.get_parameter_values(),
)
response = lambda_client.invoke(
FunctionName=self.function_name,
InvocationType=self.invocation_type,
Payload=json.dumps(payload),
Qualifier=self.qualifier,
)
success_results = dict(RequestResponse=200, Event=202, DryRun=204)
if success_results.get(
self.invocation_type) != response.get("StatusCode"):
raise Exception(
f"{self.lambda_invocation_name} failed for {self.account_id}, {self.region}"
)
else:
if response.get("FunctionError"):
error_payload = response.get("Payload").read()
raise Exception(error_payload)
else:
output = dict(
**self.params_for_results_display(),
payload=payload,
response=response,
)
self.write_output(output)
def release_spoke(puppet_account_id):
with betterboto_client.ClientContextManager(
"cloudformation",
region_name=config.get_home_region(
puppet_account_id)) as cloudformation:
cloudformation.ensure_deleted(
StackName=f"{constants.BOOTSTRAP_STACK_NAME}-spoke")
def get_ssm_parameters(self):
ssm_params = dict()
all_params = self.get_all_of_the_params()
for param_name, param_details in all_params.items():
if param_details.get("ssm"):
if param_details.get("default"):
del param_details["default"]
ssm_parameter_name = param_details.get("ssm").get("name")
ssm_parameter_name = ssm_parameter_name.replace(
"${AWS::Region}", self.region
)
ssm_parameter_name = ssm_parameter_name.replace(
"${AWS::AccountId}", self.account_id
)
ssm_params[param_name] = GetSSMParamTask(
parameter_name=param_name,
name=ssm_parameter_name,
region=param_details.get("ssm").get(
"region", config.get_home_region(self.puppet_account_id)
),
)
return ssm_params
def export_puppet_pipeline_logs(execution_id, puppet_account_id):
with betterboto_client.ClientContextManager(
"codepipeline", region_name=config.get_home_region(puppet_account_id)
) as codepipeline:
action_execution_details = codepipeline.list_action_executions(
pipelineName=constants.PIPELINE_NAME,
filter={"pipelineExecutionId": execution_id},
).get("actionExecutionDetails")
for action_execution_detail in action_execution_details:
handle_action_execution_detail(puppet_account_id, action_execution_detail)
def handle_action_execution_detail(action_execution_detail):
action_type_id = action_execution_detail.get('input').get('actionTypeId')
if action_type_id.get('category') == "Build" and action_type_id.get(
'owner') == "AWS" and action_type_id.get(
'provider') == "CodeBuild":
external_execution_id = action_execution_detail.get('output').get(
'executionResult').get('externalExecutionId')
with betterboto_client.ClientContextManager(
"codebuild",
region_name=config.get_home_region()) as codebuild:
builds = codebuild.batch_get_builds(
ids=[external_execution_id]).get('builds')
build = builds[0]
log_details = build.get('logs')
with betterboto_client.ClientContextManager(
"logs", region_name=config.get_home_region()) as logs:
with open(
f"log-{action_execution_detail.get('input').get('configuration').get('ProjectName')}.log",
'w') as f:
params = {
'logGroupName': log_details.get('groupName'),
'logStreamName': log_details.get('streamName'),
'startFromHead': True,
}
has_more_logs = True
while has_more_logs:
get_log_events_response = logs.get_log_events(**params)
if (len(get_log_events_response.get('events'))) > 0:
params['nextToken'] = get_log_events_response.get(
'nextForwardToken')
else:
has_more_logs = False
if params.get('nextToken'):
del params['nextToken']
for e in get_log_events_response.get('events'):
d = datetime.utcfromtimestamp(
e.get('timestamp') /
1000).strftime('%Y-%m-%d %H:%M:%S')
f.write(f"{d} : {e.get('message')}")
def requires(self):
ssm_params = {}
for param_name, param_details in self.parameters.items():
if param_details.get('ssm'):
if param_details.get('default'):
del param_details['default']
ssm_params[param_name] = tasks.GetSSMParamTask(
parameter_name=param_name,
name=param_details.get('ssm').get('name'),
region=param_details.get('ssm').get(
'region', config.get_home_region()))
return {
'ssm_params': ssm_params,
}
def requires(self):
all_params = {}
all_params.update(self.manifest_parameters)
all_params.update(self.launch_parameters)
all_params.update(self.account_parameters)
ssm_params = {}
for param_name, param_details in all_params.items():
if param_details.get('ssm'):
if param_details.get('default'):
del param_details['default']
ssm_params[param_name] = tasks.GetSSMParamTask(
parameter_name=param_name,
name=param_details.get('ssm').get('name'),
region=param_details.get('ssm').get(
'region', config.get_home_region()))
self.all_params = all_params
version_id = portfoliomanagement.GetVersionIdByVersionName(
self.portfolio,
self.product,
self.version,
self.account_id,
self.region,
)
product_id = portfoliomanagement.GetProductIdByProductName(
self.portfolio,
self.product,
self.account_id,
self.region,
)
return {
'ssm_params':
ssm_params,
'version':
version_id,
'product':
product_id,
'provisioning_artifact_parameters':
ProvisioningArtifactParametersTask(
self.portfolio,
self.product,
self.version,
self.account_id,
self.region,
),
}
def requires(self):
all_params = dict()
all_params.update(self.manifest_parameters)
all_params.update(self.launch_parameters)
all_params.update(self.account_parameters)
ssm_params = dict()
for param_name, param_details in all_params.items():
if param_details.get("ssm"):
if param_details.get("default"):
del param_details["default"]
ssm_params[param_name] = tasks.GetSSMParamTask(
parameter_name=param_name,
name=param_details.get("ssm").get("name"),
region=param_details.get("ssm").get(
"region", config.get_home_region(self.puppet_account_id)
),
)
self.all_params = all_params
return {
"ssm_params": ssm_params,
}
def test_get_home_region(mocked_betterboto_client):
# setup
from servicecatalog_puppet import config as sut
expected_result = {
"us-east-3",
}
mocked_response = {"Parameter": {"Value": expected_result}}
mocked_betterboto_client().__enter__(
).get_parameter.return_value = mocked_response
puppet_account_id = ""
# exercise
actual_result = sut.get_home_region(puppet_account_id)
# verify
assert actual_result == expected_result
args, kwargs = mocked_betterboto_client.call_args
assert "ssm" == args[0]
assert {} == kwargs
args, kwargs = mocked_betterboto_client().__enter__(
).get_parameter.call_args
assert args == ()
assert kwargs == {"Name": constants.HOME_REGION_PARAM_NAME}
def requires(self):
all_params = {}
all_params.update(self.manifest_parameters)
all_params.update(self.launch_parameters)
all_params.update(self.account_parameters)
ssm_params = {}
for param_name, param_details in all_params.items():
if param_details.get('ssm'):
if param_details.get('default'):
del param_details['default']
ssm_params[param_name] = tasks.GetSSMParamTask(
parameter_name=param_name,
name=param_details.get('ssm').get('name'),
region=param_details.get('ssm').get(
'region', config.get_home_region()))
self.all_params = all_params
dependencies = []
version_id = portfoliomanagement.GetVersionIdByVersionName(
self.portfolio,
self.product,
self.version,
self.account_id,
self.region,
)
product_id = portfoliomanagement.GetProductIdByProductName(
self.portfolio,
self.product,
self.account_id,
self.region,
)
for r in self.dependencies:
if r.get('status') is not None:
if r.get('status') == constants.TERMINATED:
raise Exception("Unsupported")
new_r = r.get_wrapped()
del new_r['status']
dependencies.append(self.__class__(**new_r))
else:
dependencies.append(self.__class__(**r))
return {
'dependencies':
dependencies,
'ssm_params':
ssm_params,
'version':
version_id,
'product':
product_id,
'provisioning_artifact_parameters':
ProvisioningArtifactParametersTask(
self.portfolio,
self.product,
self.version,
self.account_id,
self.region,
),
'pre_actions': [
portfoliomanagement.ProvisionActionTask(**p)
for p in self.pre_actions
],
}
def get_parameters_tasks(self):
ssm_params = dict()
boto3_params = dict()
requires = dict(
ssm_params=ssm_params,
boto3_params=boto3_params,
)
all_params = self.get_all_of_the_params()
for param_name, param_details in all_params.items():
if param_details.get("ssm"):
if param_details.get("default"):
del param_details["default"]
ssm_parameter_name = param_details.get("ssm").get("name")
ssm_parameter_name = ssm_parameter_name.replace(
"${AWS::Region}", self.region)
ssm_parameter_name = ssm_parameter_name.replace(
"${AWS::AccountId}", self.account_id)
ssm_params[param_name] = GetSSMParamTask(
parameter_name=param_name,
name=ssm_parameter_name,
region=param_details.get("ssm").get(
"region",
config.get_home_region(self.puppet_account_id)),
path=param_details.get("ssm").get("path", ""),
recursive=param_details.get("ssm").get("recursive", True),
depends_on=param_details.get("ssm").get("depends_on", []),
manifest_file_path=self.manifest_file_path,
puppet_account_id=self.puppet_account_id,
spoke_account_id=self.account_id,
spoke_region=self.region,
)
if param_details.get("boto3"):
if param_details.get("default"):
del param_details["default"]
boto3 = param_details.get("boto3")
account_id = boto3.get("account_id",
self.puppet_account_id).replace(
"${AWS::AccountId}",
self.account_id)
region = boto3.get(
"region",
config.get_home_region(self.puppet_account_id)).replace(
"${AWS::Region}", self.region)
boto3_params[param_name] = boto3_task.Boto3Task(
account_id=account_id,
region=region,
client=boto3.get("client"),
use_paginator=boto3.get("use_paginator", False),
call=boto3.get("call"),
arguments=boto3.get("arguments", {}),
filter=boto3.get("filter").replace("${AWS::Region}",
self.region).replace(
"${AWS::AccountId}",
self.account_id),
requester_task_id=self.task_id,
requester_task_family=self.task_family,
)
return requires
def requires(self):
requirements = dict()
regions = config.get_regions(self.puppet_account_id)
for launch_name, launch_details in self.manifest.get_launches_items():
portfolio = launch_details.get("portfolio")
for region in regions:
if requirements.get(region) is None:
requirements[region] = dict()
regional_details = requirements[region]
if regional_details.get(portfolio) is None:
regional_details[portfolio] = dict(products=dict())
portfolio_details = regional_details[portfolio]
if portfolio_details.get("details") is None:
portfolio_details[
"details"] = get_portfolio_by_portfolio_name_task.GetPortfolioByPortfolioName(
manifest_file_path=self.manifest_file_path,
portfolio=portfolio,
puppet_account_id=self.puppet_account_id,
account_id=self.puppet_account_id,
region=region,
)
product = launch_details.get("product")
products = portfolio_details.get("products")
if products.get(product) is None:
products[
product] = get_products_and_provisioning_artifacts_task.GetProductsAndProvisioningArtifactsTask(
manifest_file_path=self.manifest_file_path,
region=region,
portfolio=portfolio,
puppet_account_id=self.puppet_account_id,
)
params = dict()
parameter_by_paths = dict()
requirements["parameters"] = params
requirements["parameter_by_paths"] = parameter_by_paths
home_region = config.get_home_region(self.puppet_account_id)
for section in constants.SECTION_NAMES_THAT_SUPPORTS_PARAMETERS:
for item_name, item_details in self.manifest.get(section,
{}).items():
if item_details.get(
"execution") == constants.EXECUTION_MODE_SPOKE:
for parameter_name, parameter_details in item_details.get(
"parameters", {}).items():
if parameter_details.get("ssm") and str(
parameter_details.get("ssm").get(
"account_id", "")) == str(
self.puppet_account_id):
r = parameter_details.get("ssm").get(
"region",
config.get_home_region(self.puppet_account_id))
name = parameter_details.get("ssm").get("name")
path = parameter_details.get("ssm").get("path", "")
if path == "":
accounts_and_regions = self.manifest.get_account_ids_and_regions_used_for_section_item(
self.puppet_account_id, section, item_name)
for account_id, regions in accounts_and_regions.items(
):
for region in regions:
n = name.replace(
"${AWS::AccountId}",
account_id).replace(
"${AWS::Region}", region)
params[
f"{parameter_name}||{n}||{r}"] = get_ssm_param_task.GetSSMParamTask(
parameter_name=parameter_name,
name=n,
region=r,
default_value=parameter_details
.get("ssm").get(
"default_value"),
path=parameter_details.get(
"ssm").get("path", ""),
recursive=parameter_details.
get("ssm").get(
"recursive", True),
depends_on=parameter_details.
get("ssm").get(
"depends_on", []),
manifest_file_path=self.
manifest_file_path,
puppet_account_id=self.
puppet_account_id,
spoke_account_id=self.
puppet_account_id,
spoke_region=r,
)
else:
parameter_by_paths[
path] = get_ssm_param_task.GetSSMParamByPathTask(
path=parameter_details.get("ssm").get(
"path", ""),
recursive=parameter_details.get(
"ssm").get("recursive", True),
region=parameter_details.get(
"ssm").get("recursive",
home_region),
depends_on=parameter_details.get(
"ssm").get("depends_on", []),
manifest_file_path=self.
manifest_file_path,
puppet_account_id=self.
puppet_account_id,
spoke_account_id=self.
puppet_account_id,
spoke_region=home_region,
)
return requirements
def expand(f, puppet_account_id, single_account, subset=None):
click.echo("Expanding")
target_directory = os.path.sep.join([os.path.dirname(f.name), "manifests"])
assemble_manifest_from_ssm(target_directory)
manifest = manifest_utils.load(f, puppet_account_id)
org_iam_role_arn = config.get_org_iam_role_arn(puppet_account_id)
if org_iam_role_arn is None:
click.echo("No org role set - not expanding")
new_manifest = manifest
else:
click.echo("Expanding using role: {}".format(org_iam_role_arn))
with betterboto_client.CrossAccountClientContextManager(
"organizations", org_iam_role_arn, "org-iam-role"
) as client:
new_manifest = manifest_utils.expand_manifest(manifest, client)
click.echo("Expanded")
new_manifest = manifest_utils.rewrite_deploy_as_share_to_for_spoke_local_portfolios(
new_manifest
)
if single_account:
click.echo(f"Filtering for single account: {single_account}")
for account in new_manifest.get("accounts", []):
if str(account.get("account_id")) == str(single_account):
click.echo(f"Found single account: {single_account}")
new_manifest["accounts"] = [account]
break
items_to_delete = list()
for section_name in constants.ALL_SECTION_NAMES:
deploy_to_name = constants.DEPLOY_TO_NAMES[section_name]
for item_name, item in new_manifest.get(section_name, {}).items():
accounts = list()
for deploy_details in item.get(deploy_to_name, {}).get("accounts", []):
if str(deploy_details.get("account_id")) == str(single_account):
accounts.append(deploy_details)
print(f"{item_name}: there are " + str(len(accounts)))
if item.get(deploy_to_name).get("accounts"):
if len(accounts) > 0:
item[deploy_to_name]["accounts"] = accounts
else:
if item[deploy_to_name].get("tags") or item[deploy_to_name].get(
"ous"
):
del item[deploy_to_name]["accounts"]
else:
items_to_delete.append(f"{section_name}:{item_name}")
for item_to_delete in items_to_delete:
section_name, item_name = item_to_delete.split(":")
del new_manifest[section_name][item_name]
click.echo("Filtered")
new_manifest = manifest_utils.rewrite_cfct(new_manifest)
new_manifest = manifest_utils.rewrite_depends_on(new_manifest)
new_manifest = manifest_utils.rewrite_ssm_parameters(new_manifest)
new_manifest = manifest_utils.rewrite_stacks(new_manifest, puppet_account_id)
new_manifest = manifest_utils.rewrite_scps(new_manifest, puppet_account_id)
new_manifest = manifest_utils.parse_conditions(new_manifest)
if subset and subset.get("section"):
click.echo(f"Filtering for subset: {subset}")
new_manifest = manifest_utils.isolate(new_manifest, subset)
manifest_accounts_all = [
{"account_id": a.get("account_id"), "email": a.get("email")}
for a in new_manifest.get("accounts", [])
]
manifest_accounts_excluding = [
a for a in manifest_accounts_all if a.get("account_id") != puppet_account_id
]
# handle all accounts
sct_manifest_accounts = json.dumps(manifest_accounts_all)
sct_manifest_spokes = json.dumps(manifest_accounts_excluding)
regions = config.get_regions(puppet_account_id)
sct_config_regions = json.dumps(regions)
new_manifest["parameters"]["SCTManifestAccounts"] = dict(
default=sct_manifest_accounts
)
new_manifest["parameters"]["SCTManifestSpokes"] = dict(default=sct_manifest_spokes)
new_manifest["parameters"]["SCTConfigRegions"] = dict(default=sct_config_regions)
new_manifest["parameters"]["SCTAccountId"] = dict(default=puppet_account_id)
if new_manifest.get(constants.LAMBDA_INVOCATIONS) is None:
new_manifest[constants.LAMBDA_INVOCATIONS] = dict()
home_region = config.get_home_region(puppet_account_id)
with betterboto_client.ClientContextManager("ssm") as ssm:
response = ssm.get_parameter(Name="service-catalog-puppet-version")
version = response.get("Parameter").get("Value")
new_manifest["config_cache"] = dict(
home_region=home_region,
regions=regions,
should_collect_cloudformation_events=config.get_should_use_sns(
puppet_account_id, home_region
),
should_forward_events_to_eventbridge=config.get_should_use_eventbridge(
puppet_account_id, home_region
),
should_forward_failures_to_opscenter=config.get_should_forward_failures_to_opscenter(
puppet_account_id, home_region
),
puppet_version=version,
)
new_name = f.name.replace(".yaml", "-expanded.yaml")
logger.info("Writing new manifest: {}".format(new_name))
with open(new_name, "w") as output:
output.write(yaml_utils.dump(new_manifest))
def expand(f, puppet_account_id, single_account, subset=None):
click.echo("Expanding")
manifest = manifest_utils.load(f, puppet_account_id)
org_iam_role_arn = config.get_org_iam_role_arn(puppet_account_id)
if org_iam_role_arn is None:
click.echo("No org role set - not expanding")
new_manifest = manifest
else:
click.echo("Expanding using role: {}".format(org_iam_role_arn))
with betterboto_client.CrossAccountClientContextManager(
"organizations", org_iam_role_arn, "org-iam-role"
) as client:
new_manifest = manifest_utils.expand_manifest(manifest, client)
click.echo("Expanded")
if single_account:
click.echo(f"Filtering for single account: {single_account}")
for account in new_manifest.get("accounts", []):
if str(account.get("account_id")) == str(single_account):
click.echo(f"Found single account: {single_account}")
new_manifest["accounts"] = [account]
break
click.echo("Filtered")
new_manifest = manifest_utils.rewrite_depends_on(new_manifest)
new_manifest = manifest_utils.rewrite_ssm_parameters(new_manifest)
new_manifest = manifest_utils.rewrite_stacks(new_manifest, puppet_account_id)
if subset:
click.echo(f"Filtering for subset: {subset}")
new_manifest = manifest_utils.isolate(
manifest_utils.Manifest(new_manifest), subset
)
new_manifest = json.loads(json.dumps(new_manifest))
if new_manifest.get(constants.LAMBDA_INVOCATIONS) is None:
new_manifest[constants.LAMBDA_INVOCATIONS] = dict()
home_region = config.get_home_region(puppet_account_id)
with betterboto_client.ClientContextManager("ssm") as ssm:
response = ssm.get_parameter(Name="service-catalog-puppet-version")
version = response.get("Parameter").get("Value")
new_manifest["config_cache"] = dict(
home_region=home_region,
regions=config.get_regions(puppet_account_id, home_region),
should_collect_cloudformation_events=config.get_should_use_sns(
puppet_account_id, home_region
),
should_forward_events_to_eventbridge=config.get_should_use_eventbridge(
puppet_account_id, home_region
),
should_forward_failures_to_opscenter=config.get_should_forward_failures_to_opscenter(
puppet_account_id, home_region
),
puppet_version=version,
)
new_name = f.name.replace(".yaml", "-expanded.yaml")
logger.info("Writing new manifest: {}".format(new_name))
with open(new_name, "w") as output:
output.write(yaml.safe_dump(new_manifest, default_flow_style=False))
def uninstall():
with betterboto_client.ClientContextManager(
"cloudformation",
region_name=config.get_home_region()
) as cloudformation:
cloudformation.ensure_deleted(StackName=constants.BOOTSTRAP_STACK_NAME)