def test_modes(self):
report = deprecated.Report(
"some-policy",
mode=[
deprecated.field('foo', 'bar', '2021-06-30'),
deprecated.field('baz', 'yet', '2021-06-30'),
])
self.assertTrue(report)
self.assertEqual(
report.format(),
dedent("""
policy 'some-policy'
mode:
field 'foo' has been deprecated (replaced by 'bar')
field 'baz' has been deprecated (replaced by 'yet')
""")[1:-1])
def test_conditions(self):
report = deprecated.Report(
"some-policy",
conditions=[
deprecated.field('start', 'value filter in condition block',
'2021-06-30'),
deprecated.field('end', 'value filter in condition block',
'2021-06-30'),
])
self.assertTrue(report)
self.assertEqual(
report.format(),
dedent("""
policy 'some-policy'
condition:
field 'start' has been deprecated (replaced by value filter in condition block)
field 'end' has been deprecated (replaced by value filter in condition block)
""")[1:-1])
def test_field(self):
deprecation = deprecated.field('severity_normalized', 'severity_label',
'2021-06-30')
self.assertTrue(deprecation.check({'severity_normalized': '10'}))
self.assertFalse(deprecation.check({'no-match': 'ignored'}))
self.assertEqual(
str(deprecation),
"field 'severity_normalized' has been deprecated (replaced by 'severity_label')"
)
def test_footnotes(self):
footnotes = deprecated.Footnotes()
report = deprecated.Report(
"some-policy",
mode=[
deprecated.field('foo', 'bar'),
deprecated.field('baz', 'yet', '2021-06-30'),
],
actions=[
deprecated.Context(
'mark-for-op:',
deprecated.optional_fields(
('hours', 'days'),
link="http://docs.example.com/deprecations/foo#time")),
deprecated.Context(
'mark-for-op:',
deprecated.optional_field(
'tag', '2021-06-30',
"http://docs.example.com/deprecations/foo#tag")),
])
self.assertTrue(report)
self.assertEqual(report.format(footnotes=footnotes),
dedent("""
policy 'some-policy'
mode:
field 'foo' has been deprecated (replaced by 'bar')
field 'baz' has been deprecated (replaced by 'yet') [1]
actions:
mark-for-op: optional fields deprecated (one of 'hours' or 'days' must be specified) [2]
mark-for-op: optional field 'tag' deprecated (must be specified) [3]
""")[1:-1]) # noqa
self.assertEqual(footnotes(),
dedent("""
[1] Will be removed after 2021-06-30
[2] See http://docs.example.com/deprecations/foo#time
[3] See http://docs.example.com/deprecations/foo#tag, will become an error after 2021-06-30
""")[1:-1]) # noqa
class PostFinding(Action):
"""Report a finding to AWS Security Hub.
Custodian acts as a finding provider, allowing users to craft
policies that report to the AWS SecurityHub in the AWS Security Finding Format documented at
https://docs.aws.amazon.com/securityhub/latest/userguide/securityhub-findings-format.html
For resources that are taggable, we will tag the resource with an identifier
such that further findings generate updates.
Example generate a finding for accounts that don't have shield enabled.
Note with Cloud Custodian (0.9+) you need to enable the Custodian integration
to post-findings, see Getting Started with :ref:`Security Hub <aws-securityhub>`.
:example:
.. code-block:: yaml
policies:
- name: account-shield-enabled
resource: account
filters:
- shield-enabled
actions:
- type: post-finding
description: |
Shield should be enabled on account to allow for DDOS protection (1 time 3k USD Charge).
severity_label: LOW
types:
- "Software and Configuration Checks/Industry and Regulatory Standards/NIST CSF Controls (USA)"
recommendation: "Enable shield"
recommendation_url: "https://www.example.com/policies/AntiDDoS.html"
confidence: 100
compliance_status: FAILED
""" # NOQA
deprecations = (
deprecated.field('severity_normalized', 'severity_label'),
)
FindingVersion = "2018-10-08"
permissions = ('securityhub:BatchImportFindings',)
resource_type = ""
schema_alias = True
schema = type_schema(
"post-finding",
required=["types"],
title={"type": "string", 'default': 'policy.name'},
description={'type': 'string', 'default':
'policy.description, or if not defined in policy then policy.name'},
severity={"type": "number", 'default': 0},
severity_normalized={"type": "number", "min": 0, "max": 100, 'default': 0},
severity_label={
"type": "string", 'default': 'INFORMATIONAL',
"enum": ["INFORMATIONAL", "LOW", "MEDIUM", "HIGH", "CRITICAL"],
},
confidence={"type": "number", "min": 0, "max": 100},
criticality={"type": "number", "min": 0, "max": 100},
# Cross region aggregation
region={'type': 'string', 'description': 'cross-region aggregation target'},
recommendation={"type": "string"},
recommendation_url={"type": "string"},
fields={"type": "object"},
batch_size={'type': 'integer', 'minimum': 1, 'maximum': 100, 'default': 1},
types={
"type": "array",
"minItems": 1,
"items": {"type": "string"},
},
compliance_status={
"type": "string",
"enum": ["PASSED", "WARNING", "FAILED", "NOT_AVAILABLE"],
},
record_state={
"type": "string", 'default': 'ACTIVE',
"enum": ["ACTIVE", "ARCHIVED"],
},
)
NEW_FINDING = 'New'
def validate(self):
for finding_type in self.data["types"]:
if finding_type.count('/') > 2 or finding_type.split('/')[0] not in FindingTypes:
raise PolicyValidationError(
"Finding types must be in the format 'namespace/category/classifier'."
" Found {}. Valid namespace values are: {}.".format(
finding_type, " | ".join([ns for ns in FindingTypes])))
def get_finding_tag(self, resource):
finding_tag = None
tags = resource.get('Tags', [])
finding_key = '{}:{}'.format('c7n:FindingId',
self.data.get('title', self.manager.ctx.policy.name))
# Support Tags as dictionary
if isinstance(tags, dict):
return tags.get(finding_key)
# Support Tags as list of {'Key': 'Value'}
for t in tags:
key = t['Key']
value = t['Value']
if key == finding_key:
finding_tag = value
return finding_tag
def group_resources(self, resources):
grouped_resources = {}
for r in resources:
finding_tag = self.get_finding_tag(r) or self.NEW_FINDING
grouped_resources.setdefault(finding_tag, []).append(r)
return grouped_resources
def process(self, resources, event=None):
region_name = self.data.get('region', self.manager.config.region)
client = local_session(
self.manager.session_factory).client(
"securityhub", region_name=region_name)
now = datetime.now(tzutc()).isoformat()
# default batch size to one to work around security hub console issue
# which only shows a single resource in a finding.
batch_size = self.data.get('batch_size', 1)
stats = Counter()
for resource_set in chunks(resources, batch_size):
findings = []
for key, grouped_resources in self.group_resources(resource_set).items():
for resource in grouped_resources:
stats['Finding'] += 1
if key == self.NEW_FINDING:
finding_id = None
created_at = now
updated_at = now
else:
finding_id, created_at = self.get_finding_tag(
resource).split(':', 1)
updated_at = now
finding = self.get_finding(
[resource], finding_id, created_at, updated_at)
findings.append(finding)
if key == self.NEW_FINDING:
stats['New'] += 1
# Tag resources with new finding ids
tag_action = self.manager.action_registry.get('tag')
if tag_action is None:
continue
tag_action({
'key': '{}:{}'.format(
'c7n:FindingId',
self.data.get(
'title', self.manager.ctx.policy.name)),
'value': '{}:{}'.format(
finding['Id'], created_at)},
self.manager).process([resource])
else:
stats['Update'] += 1
import_response = client.batch_import_findings(
Findings=findings)
if import_response['FailedCount'] > 0:
stats['Failed'] += import_response['FailedCount']
self.log.error(
"import_response=%s" % (import_response))
self.log.debug(
"policy:%s securityhub %d findings resources %d new %d updated %d failed",
self.manager.ctx.policy.name,
stats['Finding'],
stats['New'],
stats['Update'],
stats['Failed'])
def get_finding(self, resources, existing_finding_id, created_at, updated_at):
policy = self.manager.ctx.policy
model = self.manager.resource_type
region = self.data.get('region', self.manager.config.region)
if existing_finding_id:
finding_id = existing_finding_id
else:
finding_id = '{}/{}/{}/{}'.format( # nosec
self.manager.config.region,
self.manager.config.account_id,
hashlib.md5(json.dumps( # nosemgrep
policy.data).encode('utf8')).hexdigest(),
hashlib.md5(json.dumps(list(sorted( # nosemgrep
[r[model.id] for r in resources]))).encode(
'utf8')).hexdigest())
finding = {
"SchemaVersion": self.FindingVersion,
"ProductArn": "arn:{}:securityhub:{}::product/cloud-custodian/cloud-custodian".format(
get_partition(self.manager.config.region),
region
),
"AwsAccountId": self.manager.config.account_id,
# Long search chain for description values, as this was
# made required long after users had policies deployed, so
# use explicit description, or policy description, or
# explicit title, or policy name, in that order.
"Description": self.data.get(
"description", policy.data.get(
"description",
self.data.get('title', policy.name))).strip(),
"Title": self.data.get("title", policy.name),
'Id': finding_id,
"GeneratorId": policy.name,
'CreatedAt': created_at,
'UpdatedAt': updated_at,
"RecordState": "ACTIVE",
}
severity = {'Product': 0, 'Normalized': 0, 'Label': 'INFORMATIONAL'}
if self.data.get("severity") is not None:
severity["Product"] = self.data["severity"]
if self.data.get("severity_label") is not None:
severity["Label"] = self.data["severity_label"]
# severity_normalized To be deprecated per https://docs.aws.amazon.com/securityhub/latest/userguide/securityhub-findings-format.html#asff-severity # NOQA
if self.data.get("severity_normalized") is not None:
severity["Normalized"] = self.data["severity_normalized"]
if severity:
finding["Severity"] = severity
recommendation = {}
if self.data.get("recommendation"):
recommendation["Text"] = self.data["recommendation"]
if self.data.get("recommendation_url"):
recommendation["Url"] = self.data["recommendation_url"]
if recommendation:
finding["Remediation"] = {"Recommendation": recommendation}
if "confidence" in self.data:
finding["Confidence"] = self.data["confidence"]
if "criticality" in self.data:
finding["Criticality"] = self.data["criticality"]
if "compliance_status" in self.data:
finding["Compliance"] = {"Status": self.data["compliance_status"]}
if "record_state" in self.data:
finding["RecordState"] = self.data["record_state"]
fields = {
'resource': policy.resource_type,
'ProviderName': 'CloudCustodian',
'ProviderVersion': version
}
if "fields" in self.data:
fields.update(self.data["fields"])
else:
tags = {}
for t in policy.tags:
if ":" in t:
k, v = t.split(":", 1)
else:
k, v = t, ""
tags[k] = v
fields.update(tags)
if fields:
finding["ProductFields"] = fields
finding_resources = []
for r in resources:
finding_resources.append(self.format_resource(r))
finding["Resources"] = finding_resources
finding["Types"] = list(self.data["types"])
return filter_empty(finding)
def format_envelope(self, r):
details = {}
envelope = filter_empty({
'Id': self.manager.get_arns([r])[0],
'Region': self.manager.config.region,
'Tags': {t['Key']: t['Value'] for t in r.get('Tags', [])},
'Partition': get_partition(self.manager.config.region),
'Details': {self.resource_type: details},
'Type': self.resource_type
})
return envelope, details
filter_empty = staticmethod(filter_empty)
def format_resource(self, r):
raise NotImplementedError("subclass responsibility")
class Policy:
log = logging.getLogger('custodian.policy')
deprecations = (
deprecated.field('region', 'region in condition block'),
deprecated.field('start', 'value filter in condition block'),
deprecated.field('end', 'value filter in condition block'),
)
def __init__(self, data, options, session_factory=None):
self.data = data
self.options = options
assert "name" in self.data
if session_factory is None:
session_factory = get_session_factory(self.provider_name, options)
self.session_factory = session_factory
self.ctx = ExecutionContext(self.session_factory, self, self.options)
self.resource_manager = self.load_resource_manager()
self.conditions = PolicyConditions(self, data)
def __repr__(self):
return "<Policy resource:%s name:%s region:%s>" % (
self.resource_type, self.name, self.options.region)
@property
def name(self):
return self.data['name']
@property
def resource_type(self):
return self.data['resource']
@property
def provider_name(self):
if '.' in self.resource_type:
provider_name, resource_type = self.resource_type.split('.', 1)
else:
provider_name = 'aws'
return provider_name
def is_runnable(self, event=None):
return self.conditions.evaluate(event)
# Runtime circuit breakers
@property
def max_resources(self):
return self.data.get('max-resources')
@property
def max_resources_percent(self):
return self.data.get('max-resources-percent')
@property
def tags(self):
return self.data.get('tags', ())
def get_cache(self):
return self.resource_manager._cache
@property
def execution_mode(self):
return self.data.get('mode', {'type': 'pull'})['type']
def get_execution_mode(self):
try:
exec_mode = execution[self.execution_mode]
except KeyError:
return None
return exec_mode(self)
@property
def is_lambda(self):
if 'mode' not in self.data:
return False
return True
def validate(self):
self.conditions.validate()
m = self.get_execution_mode()
if m is None:
raise PolicyValidationError("Invalid Execution mode in policy %s" %
(self.data, ))
m.validate()
self.validate_policy_start_stop()
self.resource_manager.validate()
for f in self.resource_manager.filters:
f.validate()
for a in self.resource_manager.actions:
a.validate()
def get_variables(self, variables=None):
"""Get runtime variables for policy interpolation.
Runtime variables are merged with the passed in variables
if any.
"""
# Global policy variable expansion, we have to carry forward on
# various filter/action local vocabularies. Where possible defer
# by using a format string.
#
# See https://github.com/cloud-custodian/cloud-custodian/issues/2330
if not variables:
variables = {}
partition = utils.get_partition(self.options.region)
if 'mode' in self.data:
if 'role' in self.data['mode'] and not self.data['mode'][
'role'].startswith("arn:aws"):
self.data['mode']['role'] = "arn:%s:iam::%s:role/%s" % \
(partition, self.options.account_id, self.data['mode']['role'])
variables.update({
# standard runtime variables for interpolation
'account': '{account}',
'account_id': self.options.account_id,
'partition': partition,
'region': self.options.region,
# non-standard runtime variables from local filter/action vocabularies
#
# notify action
'policy': self.data,
'event': '{event}',
# mark for op action
'op': '{op}',
'action_date': '{action_date}',
# tag action pyformat-date handling
'now': utils.FormatDate(datetime.utcnow()),
# account increase limit action
'service': '{service}',
# s3 set logging action :-( see if we can revisit this one.
'bucket_region': '{bucket_region}',
'bucket_name': '{bucket_name}',
'source_bucket_name': '{source_bucket_name}',
'source_bucket_region': '{source_bucket_region}',
'target_bucket_name': '{target_bucket_name}',
'target_prefix': '{target_prefix}',
'LoadBalancerName': '{LoadBalancerName}'
})
return variables
def expand_variables(self, variables):
"""Expand variables in policy data.
Updates the policy data in-place.
"""
# format string values returns a copy
updated = utils.format_string_values(self.data, **variables)
# Several keys should only be expanded at runtime, perserve them.
if 'member-role' in updated.get('mode', {}):
updated['mode']['member-role'] = self.data['mode']['member-role']
# Update ourselves in place
self.data = updated
# Reload filters/actions using updated data, we keep a reference
# for some compatiblity preservation work.
m = self.resource_manager
self.resource_manager = self.load_resource_manager()
# XXX: Compatiblity hack
# Preserve notify action subject lines which support
# embedded jinja2 as a passthrough to the mailer.
for old_a, new_a in zip(m.actions, self.resource_manager.actions):
if old_a.type == 'notify' and 'subject' in old_a.data:
new_a.data['subject'] = old_a.data['subject']
def push(self, event, lambda_ctx=None):
mode = self.get_execution_mode()
return mode.run(event, lambda_ctx)
def provision(self):
"""Provision policy as a lambda function."""
mode = self.get_execution_mode()
return mode.provision()
def poll(self):
"""Query resources and apply policy."""
mode = self.get_execution_mode()
return mode.run()
def get_permissions(self):
"""get permissions needed by this policy"""
permissions = set()
permissions.update(self.resource_manager.get_permissions())
for f in self.resource_manager.filters:
permissions.update(f.get_permissions())
for a in self.resource_manager.actions:
permissions.update(a.get_permissions())
return permissions
def _trim_runtime_filters(self):
from c7n.filters.core import trim_runtime
trim_runtime(self.conditions.filters)
trim_runtime(self.resource_manager.filters)
def __call__(self):
"""Run policy in default mode"""
mode = self.get_execution_mode()
if (isinstance(mode, ServerlessExecutionMode) or self.options.dryrun):
self._trim_runtime_filters()
if self.options.dryrun:
resources = PullMode(self).run()
elif not self.is_runnable():
resources = []
elif isinstance(mode, ServerlessExecutionMode):
resources = mode.provision()
else:
resources = mode.run()
# clear out resource manager post run, to clear cache
self.resource_manager = self.load_resource_manager()
return resources
run = __call__
def _write_file(self, rel_path, value):
if isinstance(self.ctx.output, NullBlobOutput):
return
with open(os.path.join(self.ctx.log_dir, rel_path), 'w') as fh:
fh.write(value)
def load_resource_manager(self):
factory = get_resource_class(self.data.get('resource'))
return factory(self.ctx, self.data)
def validate_policy_start_stop(self):
policy_name = self.data.get('name')
policy_tz = self.data.get('tz')
policy_start = self.data.get('start')
policy_end = self.data.get('end')
if policy_tz:
try:
p_tz = tzutil.gettz(policy_tz)
except Exception as e:
raise PolicyValidationError(
"Policy: %s TZ not parsable: %s, %s" %
(policy_name, policy_tz, e))
# Type will be tzwin on windows, but tzwin is null on linux
if not (isinstance(p_tz, tzutil.tzfile) or
(tzutil.tzwin and isinstance(p_tz, tzutil.tzwin))):
raise PolicyValidationError("Policy: %s TZ not parsable: %s" %
(policy_name, policy_tz))
for i in [policy_start, policy_end]:
if i:
try:
parser.parse(i)
except Exception as e:
raise ValueError(
"Policy: %s Date/Time not parsable: %s, %s" %
(policy_name, i, e))
def get_deprecations(self):
"""Return any matching deprecations for the policy fields itself."""
return deprecated.check_deprecations(self, "policy")