def validate_cfn_coeffect(world):
cfn_coeffect = get_in(['coeffects', 'cloudformation'], world)
try:
return update_in(world, ['valid_cfn'],
lambda x: merge(x or {}, CFN_COEFFECT_SCHEMA(cfn_coeffect)))
except Invalid:
logger.warning(cfn_coeffect)
logger.warning('CloudFormation Coeffects are not valid; using defaults', exc_info=True)
return update_in(world, ['valid_cfn'],
lambda x: merge(x or {}, DEFAULT_CFN_COEFFECT))
def vega2to3(data):
"""Transform a Vega data list from version 2 to 3.
Args:
data: vega data list
Returns:
update vega data list
"""
def keymapping(key):
"""Map vega data keys from version 2 to 3
The mapping is `test` -> `expr` and `field` -> `as` otherwise
the input key is just returned.
Args:
key: the key to map
Returns:
a new key
"""
return dict(test='expr', field='as').get(key, key)
update_transform = fcompose(map(keymap(keymapping)), list)
return pipe(
data,
map(update_in(keys=['transform'], func=update_transform, default=[])),
list)
def discover_master_payer_account(world):
report_definitions = coeffects_report_definitions(world)
logger.info(f'Found these ReportDefinitions: {report_definitions}')
local_buckets = {x['Name'] for x in coeffects_buckets(world)}
valid_report_definitions = keep_valid(MINIMUM_BILLING_REPORT,
report_definitions)
logger.info(
f'Found these _valid_ ReportDefinitions: {valid_report_definitions}')
first_valid = get_first_valid_report_definition(valid_report_definitions,
default={})
valid_local_report_definitions = [
x for x in valid_report_definitions if x['S3Bucket'] in local_buckets
]
logger.info(
f'Found these _valid local_ ReportDefinitions: {valid_local_report_definitions}'
)
local = any(valid_local_report_definitions)
first_valid_local = get_first_valid_report_definition(
valid_local_report_definitions, default=first_valid)
bucket_name = first_valid_local.get('S3Bucket')
bucket_path = f"{first_valid_local.get('S3Prefix', '')}/{first_valid_local.get('ReportName', '')}" if bucket_name else None
output = {
'IsMasterPayerAccount': local,
'MasterPayerBillingBucketName': bucket_name,
'MasterPayerBillingBucketPath': bucket_path,
}
return update_in(world, ['output'], lambda x: merge(x or {}, output))
def filter_data(field, yaml_data):
"""Extract a field of data from the YAML files.
Args:
field: the name of the field to extract
yaml_data: the benchmark YAML data
Returns:
the filtered data from the YAML data
"""
return pipe(
yaml_data,
dict,
valmap(lambda val: val["data"]),
valmap(filter(lambda item: item["name"].lower() == field)),
valmap(list),
valmap(get(0, default=None)),
valfilter(lambda x: x is not None),
itemmap(lambda item: (item[0], update_dict(item[1], name=item[0]))),
lambda dict_: sorted(list(dict_.values()), key=lambda item: item["name"]),
map(
update_in(
keys=["transform"],
func=lambda x: x + [dict(expr="datum.x > 0.01", type="filter")],
)
),
)
def discover_organization_master_account(world):
account_id = event_account_id(world)
master_account_id = coeffects_master_account_id(world)
output = {
'IsOrganizationMasterAccount': account_id == master_account_id,
}
return update_in(world, ['output'], lambda x: merge(x or {}, output))
def get_bag(build_dir: Path, base_dtype: str = "xml") -> db.Bag:
"""possible to do any text pre-processing here"""
dtype_path = get_datapaths(build_dir).get(base_dtype)
schema = get_schema(build_dir)
filepaths = dtype_path.glob(f"**/*.{base_dtype}")
_update_authors = flip(update_in(func=flatten_authors), ("authors", ))
_update_keywords = lambda d: pipe(
d,
*[
update_in(func=split_keywords, keys=[kw])
for kw in (col for col in d.keys() if col.endswith("_keywords"))
],
)
return (db.from_sequence(filepaths).map(partial(
load_xml, schema)).map(_update_authors).map(_update_keywords))
def discover_audit_account(world):
trail = get_first_valid_trail(world)
trail_bucket = trail.get('S3BucketName')
local_buckets = {x['Name'] for x in coeffects_buckets(world)}
output = {
'IsAuditAccount': trail_bucket in local_buckets,
'RemoteCloudTrailBucket': not (trail_bucket in local_buckets),
'AuditCloudTrailBucketName': trail_bucket,
'AuditCloudTrailBucketPrefix': trail.get('S3KeyPrefix'),
}
return update_in(world, ['output'], lambda x: merge(x or {}, output))
def prepare_output(world):
valid_cfn = get_in(['valid_cfn'], world)
metadata = callback_metadata(properties(world))
message_type = 'account-link-provisioned' if request_type(world) in {'Create', 'Update'} else 'account-link-deprovisioned'
visible_cloudtrail_arns_string = null_to_none(get_in(['Discovery', 'VisibleCloudTrailArns'], valid_cfn))
visible_cloudtrail_arns = visible_cloudtrail_arns_string.split(',') if visible_cloudtrail_arns_string else None
master_payer_billing_bucket_name = (null_to_none(get_in(['Discovery', 'MasterPayerBillingBucketName'], valid_cfn)) or
null_to_none(get_in(['MasterPayerAccount', 'ReportS3Bucket'], valid_cfn)))
master_payer_billing_bucket_path = (null_to_none(get_in(['Discovery', 'MasterPayerBillingBucketPath'], valid_cfn)) or
null_to_none(get_in(['MasterPayerAccount', 'ReportS3Prefix'], valid_cfn)))
output = {
**default_metadata,
'message_type': message_type,
'data': {
'metadata': {
'cloud_region': metadata['Region'],
'external_id': metadata['ExternalId'],
'cloud_account_id': metadata['AccountId'],
'cz_account_name': metadata['AccountName'],
'reactor_id': metadata['ReactorId'],
'reactor_callback_url': metadata['ReactorCallbackUrl'],
},
'links': {
'audit': {'role_arn': null_to_none(get_in(['AuditAccount', 'RoleArn'], valid_cfn))},
'cloudtrail_owner': {
'sqs_queue_arn': null_to_none(get_in(['CloudTrailOwnerAccount', 'SQSQueueArn'], valid_cfn)),
'sqs_queue_policy_name': null_to_none(get_in(['CloudTrailOwnerAccount', 'SQSQueuePolicyName'], valid_cfn)),
},
'master_payer': {'role_arn': null_to_none(get_in(['MasterPayerAccount', 'RoleArn'], valid_cfn))},
'resource_owner': {'role_arn': null_to_none(get_in(['ResourceOwnerAccount', 'RoleArn'], valid_cfn))},
'legacy': {'role_arn': null_to_none(get_in(['LegacyAccount', 'RoleArn'], valid_cfn))},
},
'discovery': {
'audit_cloudtrail_bucket_name': null_to_none(get_in(['Discovery', 'AuditCloudTrailBucketName'], valid_cfn)),
'audit_cloudtrail_bucket_prefix': null_to_none(get_in(['Discovery', 'AuditCloudTrailBucketPrefix'], valid_cfn)),
'cloudtrail_sns_topic_arn': null_to_none(get_in(['Discovery', 'CloudTrailSNSTopicArn'], valid_cfn)),
'cloudtrail_trail_arn': null_to_none(get_in(['Discovery', 'CloudTrailTrailArn'], valid_cfn)),
'is_audit_account': string_to_bool(get_in(['Discovery', 'IsAuditAccount'], valid_cfn)),
'is_cloudtrail_owner_account': string_to_bool(get_in(['Discovery', 'IsCloudTrailOwnerAccount'], valid_cfn)),
'is_master_payer_account': string_to_bool(get_in(['Discovery', 'IsMasterPayerAccount'], valid_cfn)),
'is_organization_master_account': string_to_bool(get_in(['Discovery', 'IsOrganizationMasterAccount'], valid_cfn)),
'is_organization_trail': string_to_bool(get_in(['Discovery', 'IsOrganizationTrail'], valid_cfn)),
'is_resource_owner_account': string_to_bool(get_in(['Discovery', 'IsResourceOwnerAccount'], valid_cfn)),
'master_payer_billing_bucket_name': master_payer_billing_bucket_name,
'master_payer_billing_bucket_path': master_payer_billing_bucket_path,
'remote_cloudtrail_bucket': string_to_bool(get_in(['Discovery', 'RemoteCloudTrailBucket'], valid_cfn)),
'visible_cloudtrail_arns': visible_cloudtrail_arns,
}
}
}
return update_in(world, ['output'], lambda x: merge(x or {}, output))
def discover_cloudtrail_account(world):
visible_trails = get_visible_cloudtrail_arns(world)
trail = get_first_valid_trail(world)
trail_topic = trail.get('SnsTopicARN')
account_id = trail_topic.split(':')[4] if trail_topic else None
output = {
'IsCloudTrailOwnerAccount': account_id == event_account_id(world),
'IsOrganizationTrail': trail.get('IsOrganizationTrail'),
'CloudTrailSNSTopicArn': trail_topic,
'CloudTrailTrailArn': trail.get('TrailARN'),
'VisibleCloudTrailArns': visible_trails,
}
return update_in(world, ['output'], lambda x: merge(x or {}, output))
def discover_master_payer_account(world):
is_account_not_in_organization = output_is_account_outside_organization(
world)
is_account_organization_master_account = output_is_organization_master(
world)
is_master_payer = is_account_not_in_organization or is_account_organization_master_account
report_definitions = coeffects_payer_reports(world)['report_definitions']
(bucket_name,
bucket_path) = get_cur_bucket_if_local(world, report_definitions)
output = {
'IsMasterPayerAccount': is_master_payer,
'MasterPayerBillingBucketName': bucket_name,
'MasterPayerBillingBucketPath': bucket_path,
}
return update_in(world, ['output'], lambda x: merge(x or {}, output))
def test_default_stack(snapshot):
logger = DummyLogger()
sender = build_sender(middleware=get_test_stack(logger))
sender("get", "http://example.com")
snapshot.assert_match(
list(
map(
lambda arr: [
arr[0],
arr[1],
sanitize(update_in(arr[2], ["headers"], sanitize)),
sanitize(arr[3]),
],
logger.messages,
)))
snapshot.assert_match(sanitize(logger.context))
def discover_connected_account(world):
output = {
'IsResourceOwnerAccount': True,
}
return update_in(world, ['output'], lambda x: merge(x or {}, output))
