def validate_cfn_coeffect(world): cfn_coeffect = get_in(['coeffects', 'cloudformation'], world) try: return update_in(world, ['valid_cfn'], lambda x: merge(x or {}, CFN_COEFFECT_SCHEMA(cfn_coeffect))) except Invalid: logger.warning(cfn_coeffect) logger.warning('CloudFormation Coeffects are not valid; using defaults', exc_info=True) return update_in(world, ['valid_cfn'], lambda x: merge(x or {}, DEFAULT_CFN_COEFFECT))
def vega2to3(data): """Transform a Vega data list from version 2 to 3. Args: data: vega data list Returns: update vega data list """ def keymapping(key): """Map vega data keys from version 2 to 3 The mapping is `test` -> `expr` and `field` -> `as` otherwise the input key is just returned. Args: key: the key to map Returns: a new key """ return dict(test='expr', field='as').get(key, key) update_transform = fcompose(map(keymap(keymapping)), list) return pipe( data, map(update_in(keys=['transform'], func=update_transform, default=[])), list)
def discover_master_payer_account(world): report_definitions = coeffects_report_definitions(world) logger.info(f'Found these ReportDefinitions: {report_definitions}') local_buckets = {x['Name'] for x in coeffects_buckets(world)} valid_report_definitions = keep_valid(MINIMUM_BILLING_REPORT, report_definitions) logger.info( f'Found these _valid_ ReportDefinitions: {valid_report_definitions}') first_valid = get_first_valid_report_definition(valid_report_definitions, default={}) valid_local_report_definitions = [ x for x in valid_report_definitions if x['S3Bucket'] in local_buckets ] logger.info( f'Found these _valid local_ ReportDefinitions: {valid_local_report_definitions}' ) local = any(valid_local_report_definitions) first_valid_local = get_first_valid_report_definition( valid_local_report_definitions, default=first_valid) bucket_name = first_valid_local.get('S3Bucket') bucket_path = f"{first_valid_local.get('S3Prefix', '')}/{first_valid_local.get('ReportName', '')}" if bucket_name else None output = { 'IsMasterPayerAccount': local, 'MasterPayerBillingBucketName': bucket_name, 'MasterPayerBillingBucketPath': bucket_path, } return update_in(world, ['output'], lambda x: merge(x or {}, output))
def filter_data(field, yaml_data): """Extract a field of data from the YAML files. Args: field: the name of the field to extract yaml_data: the benchmark YAML data Returns: the filtered data from the YAML data """ return pipe( yaml_data, dict, valmap(lambda val: val["data"]), valmap(filter(lambda item: item["name"].lower() == field)), valmap(list), valmap(get(0, default=None)), valfilter(lambda x: x is not None), itemmap(lambda item: (item[0], update_dict(item[1], name=item[0]))), lambda dict_: sorted(list(dict_.values()), key=lambda item: item["name"]), map( update_in( keys=["transform"], func=lambda x: x + [dict(expr="datum.x > 0.01", type="filter")], ) ), )
def discover_organization_master_account(world): account_id = event_account_id(world) master_account_id = coeffects_master_account_id(world) output = { 'IsOrganizationMasterAccount': account_id == master_account_id, } return update_in(world, ['output'], lambda x: merge(x or {}, output))
def get_bag(build_dir: Path, base_dtype: str = "xml") -> db.Bag: """possible to do any text pre-processing here""" dtype_path = get_datapaths(build_dir).get(base_dtype) schema = get_schema(build_dir) filepaths = dtype_path.glob(f"**/*.{base_dtype}") _update_authors = flip(update_in(func=flatten_authors), ("authors", )) _update_keywords = lambda d: pipe( d, *[ update_in(func=split_keywords, keys=[kw]) for kw in (col for col in d.keys() if col.endswith("_keywords")) ], ) return (db.from_sequence(filepaths).map(partial( load_xml, schema)).map(_update_authors).map(_update_keywords))
def discover_audit_account(world): trail = get_first_valid_trail(world) trail_bucket = trail.get('S3BucketName') local_buckets = {x['Name'] for x in coeffects_buckets(world)} output = { 'IsAuditAccount': trail_bucket in local_buckets, 'RemoteCloudTrailBucket': not (trail_bucket in local_buckets), 'AuditCloudTrailBucketName': trail_bucket, 'AuditCloudTrailBucketPrefix': trail.get('S3KeyPrefix'), } return update_in(world, ['output'], lambda x: merge(x or {}, output))
def prepare_output(world): valid_cfn = get_in(['valid_cfn'], world) metadata = callback_metadata(properties(world)) message_type = 'account-link-provisioned' if request_type(world) in {'Create', 'Update'} else 'account-link-deprovisioned' visible_cloudtrail_arns_string = null_to_none(get_in(['Discovery', 'VisibleCloudTrailArns'], valid_cfn)) visible_cloudtrail_arns = visible_cloudtrail_arns_string.split(',') if visible_cloudtrail_arns_string else None master_payer_billing_bucket_name = (null_to_none(get_in(['Discovery', 'MasterPayerBillingBucketName'], valid_cfn)) or null_to_none(get_in(['MasterPayerAccount', 'ReportS3Bucket'], valid_cfn))) master_payer_billing_bucket_path = (null_to_none(get_in(['Discovery', 'MasterPayerBillingBucketPath'], valid_cfn)) or null_to_none(get_in(['MasterPayerAccount', 'ReportS3Prefix'], valid_cfn))) output = { **default_metadata, 'message_type': message_type, 'data': { 'metadata': { 'cloud_region': metadata['Region'], 'external_id': metadata['ExternalId'], 'cloud_account_id': metadata['AccountId'], 'cz_account_name': metadata['AccountName'], 'reactor_id': metadata['ReactorId'], 'reactor_callback_url': metadata['ReactorCallbackUrl'], }, 'links': { 'audit': {'role_arn': null_to_none(get_in(['AuditAccount', 'RoleArn'], valid_cfn))}, 'cloudtrail_owner': { 'sqs_queue_arn': null_to_none(get_in(['CloudTrailOwnerAccount', 'SQSQueueArn'], valid_cfn)), 'sqs_queue_policy_name': null_to_none(get_in(['CloudTrailOwnerAccount', 'SQSQueuePolicyName'], valid_cfn)), }, 'master_payer': {'role_arn': null_to_none(get_in(['MasterPayerAccount', 'RoleArn'], valid_cfn))}, 'resource_owner': {'role_arn': null_to_none(get_in(['ResourceOwnerAccount', 'RoleArn'], valid_cfn))}, 'legacy': {'role_arn': null_to_none(get_in(['LegacyAccount', 'RoleArn'], valid_cfn))}, }, 'discovery': { 'audit_cloudtrail_bucket_name': null_to_none(get_in(['Discovery', 'AuditCloudTrailBucketName'], valid_cfn)), 'audit_cloudtrail_bucket_prefix': null_to_none(get_in(['Discovery', 'AuditCloudTrailBucketPrefix'], valid_cfn)), 'cloudtrail_sns_topic_arn': null_to_none(get_in(['Discovery', 'CloudTrailSNSTopicArn'], valid_cfn)), 'cloudtrail_trail_arn': null_to_none(get_in(['Discovery', 'CloudTrailTrailArn'], valid_cfn)), 'is_audit_account': string_to_bool(get_in(['Discovery', 'IsAuditAccount'], valid_cfn)), 'is_cloudtrail_owner_account': string_to_bool(get_in(['Discovery', 'IsCloudTrailOwnerAccount'], valid_cfn)), 'is_master_payer_account': string_to_bool(get_in(['Discovery', 'IsMasterPayerAccount'], valid_cfn)), 'is_organization_master_account': string_to_bool(get_in(['Discovery', 'IsOrganizationMasterAccount'], valid_cfn)), 'is_organization_trail': string_to_bool(get_in(['Discovery', 'IsOrganizationTrail'], valid_cfn)), 'is_resource_owner_account': string_to_bool(get_in(['Discovery', 'IsResourceOwnerAccount'], valid_cfn)), 'master_payer_billing_bucket_name': master_payer_billing_bucket_name, 'master_payer_billing_bucket_path': master_payer_billing_bucket_path, 'remote_cloudtrail_bucket': string_to_bool(get_in(['Discovery', 'RemoteCloudTrailBucket'], valid_cfn)), 'visible_cloudtrail_arns': visible_cloudtrail_arns, } } } return update_in(world, ['output'], lambda x: merge(x or {}, output))
def discover_cloudtrail_account(world): visible_trails = get_visible_cloudtrail_arns(world) trail = get_first_valid_trail(world) trail_topic = trail.get('SnsTopicARN') account_id = trail_topic.split(':')[4] if trail_topic else None output = { 'IsCloudTrailOwnerAccount': account_id == event_account_id(world), 'IsOrganizationTrail': trail.get('IsOrganizationTrail'), 'CloudTrailSNSTopicArn': trail_topic, 'CloudTrailTrailArn': trail.get('TrailARN'), 'VisibleCloudTrailArns': visible_trails, } return update_in(world, ['output'], lambda x: merge(x or {}, output))
def discover_master_payer_account(world): is_account_not_in_organization = output_is_account_outside_organization( world) is_account_organization_master_account = output_is_organization_master( world) is_master_payer = is_account_not_in_organization or is_account_organization_master_account report_definitions = coeffects_payer_reports(world)['report_definitions'] (bucket_name, bucket_path) = get_cur_bucket_if_local(world, report_definitions) output = { 'IsMasterPayerAccount': is_master_payer, 'MasterPayerBillingBucketName': bucket_name, 'MasterPayerBillingBucketPath': bucket_path, } return update_in(world, ['output'], lambda x: merge(x or {}, output))
def test_default_stack(snapshot): logger = DummyLogger() sender = build_sender(middleware=get_test_stack(logger)) sender("get", "http://example.com") snapshot.assert_match( list( map( lambda arr: [ arr[0], arr[1], sanitize(update_in(arr[2], ["headers"], sanitize)), sanitize(arr[3]), ], logger.messages, ))) snapshot.assert_match(sanitize(logger.context))
def discover_connected_account(world): output = { 'IsResourceOwnerAccount': True, } return update_in(world, ['output'], lambda x: merge(x or {}, output))