class GCPMetricsFilter(Filter):
"""Supports metrics filters on resources.
All resources that have cloud watch metrics are supported.
Docs on cloud watch metrics
- Google Supported Metrics
https://cloud.google.com/monitoring/api/metrics_gcp
- Custom Metrics
https://cloud.google.com/monitoring/api/v3/metric-model#intro-custom-metrics
.. code-block:: yaml
- name: firewall-hit-count
resource: gcp.firewall
filters:
- type: metrics
name: firewallinsights.googleapis.com/subnet/firewall_hit_count
aligner: ALIGN_COUNT
days: 14
value: 1
op: greater-than
"""
schema = type_schema(
'metrics', **{
'name': {
'type': 'string'
},
'metric-key': {
'type': 'string'
},
'group-by-fields': {
'type': 'array',
'items': {
'type': 'string'
}
},
'days': {
'type': 'number'
},
'op': {
'type': 'string',
'enum': list(OPERATORS.keys())
},
'reducer': {
'type': 'string',
'enum': REDUCERS
},
'aligner': {
'type': 'string',
'enum': ALIGNERS
},
'value': {
'type': 'number'
},
'filter': {
'type': 'string'
},
'missing-value': {
'type': 'number'
},
'required': ('value', 'name', 'op')
})
permissions = ("monitoring.timeSeries.list", )
def validate(self):
if not self.data.get('metric-key') and \
not hasattr(self.manager.resource_type, 'metric_key'):
raise FilterValidationError(
"metric-key not defined for resource %s,"
"so must be provided in the policy" % (self.manager.type))
return self
def process(self, resources, event=None):
days = self.data.get('days', 14)
duration = timedelta(days)
self.metric = self.data['name']
self.metric_key = self.data.get(
'metric-key') or self.manager.resource_type.metric_key
self.aligner = self.data.get('aligner', 'ALIGN_NONE')
self.reducer = self.data.get('reducer', 'REDUCE_NONE')
self.group_by_fields = self.data.get('group-by-fields', [])
self.missing_value = self.data.get('missing-value')
self.end = datetime.now(pytz.timezone('UTC'))
self.start = self.end - duration
self.period = str((self.end - self.start).total_seconds()) + 's'
self.resource_metric_dict = {}
self.op = OPERATORS[self.data.get('op', 'less-than')]
self.value = self.data['value']
self.filter = self.data.get('filter', '')
self.c7n_metric_key = "%s.%s.%s" % (self.metric, self.aligner,
self.reducer)
session = local_session(self.manager.session_factory)
client = session.client("monitoring", "v3", "projects.timeSeries")
project = session.get_default_project()
time_series_data = []
for batched_filter in self.get_batched_query_filter(resources):
query_params = {
'filter': batched_filter,
'interval_startTime': self.start.isoformat(),
'interval_endTime': self.end.isoformat(),
'aggregation_alignmentPeriod': self.period,
"aggregation_perSeriesAligner": self.aligner,
"aggregation_crossSeriesReducer": self.reducer,
"aggregation_groupByFields": self.group_by_fields,
'view': 'FULL'
}
metric_list = client.execute_query('list', {
'name': 'projects/' + project,
**query_params
})
time_series_data.extend(metric_list.get('timeSeries', []))
if not time_series_data:
self.log.info("No metrics found for {}".format(
self.c7n_metric_key))
return []
self.split_by_resource(time_series_data)
matched = [r for r in resources if self.process_resource(r)]
return matched
def batch_resources(self, resources):
if not resources:
return []
batched_resources = []
resource_filter = []
batch_size = len(self.filter)
for r in resources:
resource_name = self.manager.resource_type.get_metric_resource_name(
r)
resource_filter_item = '{} = "{}"'.format(self.metric_key,
resource_name)
resource_filter.append(resource_filter_item)
resource_filter.append(' OR ')
batch_size += len(resource_filter_item) + 4
if batch_size >= BATCH_SIZE:
resource_filter.pop()
batched_resources.append(resource_filter)
resource_filter = []
batch_size = len(self.filter)
resource_filter.pop()
batched_resources.append(resource_filter)
return batched_resources
def get_batched_query_filter(self, resources):
batched_filters = []
metric_filter_type = 'metric.type = "{}" AND ( '.format(self.metric)
user_filter = ''
if self.filter:
user_filter = " AND " + self.filter
for batch in self.batch_resources(resources):
batched_filters.append(''.join(
[metric_filter_type, ''.join(batch), ' ) ', user_filter]))
return batched_filters
def split_by_resource(self, metric_list):
for m in metric_list:
resource_name = jmespath.search(self.metric_key, m)
self.resource_metric_dict[resource_name] = m
def process_resource(self, resource):
resource_metric = resource.setdefault('c7n.metrics', {})
resource_name = self.manager.resource_type.get_metric_resource_name(
resource)
metric = self.resource_metric_dict.get(resource_name)
if not metric and not self.missing_value:
return False
if not metric:
metric_value = self.missing_value
else:
metric_value = float(
list(metric["points"][0]["value"].values())[0])
resource_metric[self.c7n_metric_key] = metric
matched = self.op(metric_value, self.value)
return matched
@classmethod
def register_resources(klass, registry, resource_class):
resource_class.filter_registry.register('metrics', klass)
def generate(resource_types=()):
resource_defs = {}
definitions = {
'resources': resource_defs,
'iam-statement': {
'additionalProperties': False,
'type': 'object',
'properties': {
'Sid': {'type': 'string'},
'Effect': {'type': 'string', 'enum': ['Allow', 'Deny']},
'Principal': {'anyOf': [
{'type': 'string'},
{'type': 'object'}, {'type': 'array'}]},
'NotPrincipal': {'anyOf': [{'type': 'object'}, {'type': 'array'}]},
'Action': {'anyOf': [{'type': 'string'}, {'type': 'array'}]},
'NotAction': {'anyOf': [{'type': 'string'}, {'type': 'array'}]},
'Resource': {'anyOf': [{'type': 'string'}, {'type': 'array'}]},
'NotResource': {'anyOf': [{'type': 'string'}, {'type': 'array'}]},
'Condition': {'type': 'object'}
},
'required': ['Sid', 'Effect'],
'oneOf': [
{'required': ['Principal', 'Action', 'Resource']},
{'required': ['NotPrincipal', 'Action', 'Resource']},
{'required': ['Principal', 'NotAction', 'Resource']},
{'required': ['NotPrincipal', 'NotAction', 'Resource']},
{'required': ['Principal', 'Action', 'NotResource']},
{'required': ['NotPrincipal', 'Action', 'NotResource']},
{'required': ['Principal', 'NotAction', 'NotResource']},
{'required': ['NotPrincipal', 'NotAction', 'NotResource']}
]
},
'actions': {},
'filters': {
'value': ValueFilter.schema,
'event': EventFilter.schema,
'age': AgeFilter.schema,
# Shortcut form of value filter as k=v
'valuekv': {
'type': 'object',
'additionalProperties': {'oneOf': [{'type': 'number'}, {'type': 'null'},
{'type': 'array', 'maxItems': 0}, {'type': 'string'}, {'type': 'boolean'}]},
'minProperties': 1,
'maxProperties': 1},
},
'filters_common': {
'comparison_operators': {
'enum': list(OPERATORS.keys())},
'value_types': {'enum': VALUE_TYPES},
'value_from': ValuesFrom.schema,
'value': {'oneOf': [
{'type': 'array'},
{'type': 'string'},
{'type': 'boolean'},
{'type': 'number'},
{'type': 'null'}]},
},
'policy': {
'type': 'object',
'required': ['name', 'resource'],
'additionalProperties': False,
'properties': {
'name': {
'type': 'string',
'pattern': "^[A-z][A-z0-9]*(-[A-z0-9]+)*$"},
'region': {'type': 'string'},
'tz': {'type': 'string'},
'start': {'format': 'date-time'},
'end': {'format': 'date-time'},
'resource': {'type': 'string'},
'max-resources': {'anyOf': [
{'type': 'integer', 'minimum': 1},
{'$ref': '#/definitions/max-resources-properties'}
]},
'max-resources-percent': {'type': 'number', 'minimum': 0, 'maximum': 100},
'comment': {'type': 'string'},
'comments': {'type': 'string'},
'description': {'type': 'string'},
'tags': {'type': 'array', 'items': {'type': 'string'}},
'mode': {'$ref': '#/definitions/policy-mode'},
'source': {'enum': ['describe', 'config', 'resource-graph']},
'actions': {
'type': 'array',
},
'filters': {
'type': 'array'
},
#
# TODO: source queries should really move under
# source. This was initially used for describe sources
# to expose server side query mechanisms, however its
# important to note it also prevents resource cache
# utilization between policies that have different
# queries.
'query': {
'type': 'array', 'items': {'type': 'object'}}
},
},
'policy-mode': {
'anyOf': [e.schema for _, e in execution.items()],
},
'max-resources-properties': {
'type': 'object',
'additionalProperties': False,
'properties': {
'amount': {"type": 'integer', 'minimum': 1},
'op': {'enum': ['or', 'and']},
'percent': {'type': 'number', 'minimum': 0, 'maximum': 100}
}
}
}
resource_refs = []
for cloud_name, cloud_type in clouds.items():
for type_name, resource_type in cloud_type.resources.items():
r_type_name = "%s.%s" % (cloud_name, type_name)
if resource_types and r_type_name not in resource_types:
if not resource_type.type_aliases:
continue
# atm only azure is using type aliases.
elif not set([
"%s.%s" % (cloud_name, ralias) for ralias
in resource_type.type_aliases]).intersection(
resource_types):
continue
aliases = []
if resource_type.type_aliases:
aliases.extend(["%s.%s" % (cloud_name, a) for a in resource_type.type_aliases])
# aws gets legacy aliases with no cloud prefix
if cloud_name == 'aws':
aliases.extend(resource_type.type_aliases)
# aws gets additional alias for default name
if cloud_name == 'aws':
aliases.append(type_name)
resource_refs.append(
process_resource(
r_type_name,
resource_type,
resource_defs,
aliases,
definitions,
cloud_name
))
schema = {
"$schema": "http://json-schema.org/draft-07/schema#",
'id': 'http://schema.cloudcustodian.io/v0/custodian.json',
'definitions': definitions,
'type': 'object',
'required': ['policies'],
'additionalProperties': False,
'properties': {
'vars': {'type': 'object'},
'policies': {
'type': 'array',
'additionalItems': False,
'items': {'anyOf': resource_refs}
}
}
}
return schema
class MetricsFilter(Filter):
"""Supports cloud watch metrics filters on resources.
All resources that have cloud watch metrics are supported.
Docs on cloud watch metrics
- GetMetricStatistics - http://goo.gl/w8mMEY
- Supported Metrics - http://goo.gl/n0E0L7
.. code-block:: yaml
- name: ec2-underutilized
resource: ec2
filters:
- type: metrics
name: CPUUtilization
days: 4
period: 86400
value: 30
op: less-than
Note periods when a resource is not sending metrics are not part
of calculated statistics as in the case of a stopped ec2 instance,
nor for resources to new to have existed the entire
period. ie. being stopped for an ec2 instance wouldn't lower the
average cpu utilization.
Note the default statistic for metrics is Average.
"""
schema = type_schema(
'metrics',
**{
'namespace': {
'type': 'string'
},
'name': {
'type': 'string'
},
'dimensions': {
'type': 'array',
'items': {
'type': 'string'
}
},
# Type choices
'statistics': {
'type': 'string',
'enum':
['Average', 'Sum', 'Maximum', 'Minimum', 'SampleCount']
},
'days': {
'type': 'number'
},
'op': {
'type': 'string',
'enum': list(OPERATORS.keys())
},
'value': {
'type': 'number'
},
'period': {
'type': 'number'
},
'attr-multiplier': {
'type': 'number'
},
'percent-attr': {
'type': 'string'
},
'required': ('value', 'name')
})
schema_alias = True
permissions = ("cloudwatch:GetMetricStatistics", )
MAX_QUERY_POINTS = 50850
MAX_RESULT_POINTS = 1440
# Default per service, for overloaded services like ec2
# we do type specific default namespace annotation
# specifically AWS/EBS and AWS/EC2Spot
# ditto for spot fleet
DEFAULT_NAMESPACE = {
'cloudfront': 'AWS/CloudFront',
'cloudsearch': 'AWS/CloudSearch',
'dynamodb': 'AWS/DynamoDB',
'ecs': 'AWS/ECS',
'elasticache': 'AWS/ElastiCache',
'ec2': 'AWS/EC2',
'elb': 'AWS/ELB',
'elbv2': 'AWS/ApplicationELB',
'emr': 'AWS/EMR',
'es': 'AWS/ES',
'events': 'AWS/Events',
'firehose': 'AWS/Firehose',
'kinesis': 'AWS/Kinesis',
'lambda': 'AWS/Lambda',
'logs': 'AWS/Logs',
'redshift': 'AWS/Redshift',
'rds': 'AWS/RDS',
'route53': 'AWS/Route53',
's3': 'AWS/S3',
'sns': 'AWS/SNS',
'sqs': 'AWS/SQS',
}
def process(self, resources, event=None):
days = self.data.get('days', 14)
duration = timedelta(days)
self.metric = self.data['name']
self.end = datetime.utcnow()
self.start = self.end - duration
self.period = int(self.data.get('period', duration.total_seconds()))
self.statistics = self.data.get('statistics', 'Average')
self.model = self.manager.get_model()
self.op = OPERATORS[self.data.get('op', 'less-than')]
self.value = self.data['value']
ns = self.data.get('namespace')
if not ns:
ns = getattr(self.model, 'metrics_namespace', None)
if not ns:
ns = self.DEFAULT_NAMESPACE[self.model.service]
self.namespace = ns
self.log.debug("Querying metrics for %d", len(resources))
matched = []
with self.executor_factory(max_workers=3) as w:
futures = []
for resource_set in chunks(resources, 50):
futures.append(
w.submit(self.process_resource_set, resource_set))
for f in as_completed(futures):
if f.exception():
self.log.warning("CW Retrieval error: %s" % f.exception())
continue
matched.extend(f.result())
return matched
def get_dimensions(self, resource):
return [{
'Name': self.model.dimension,
'Value': resource[self.model.dimension]
}]
def process_resource_set(self, resource_set):
client = local_session(
self.manager.session_factory).client('cloudwatch')
matched = []
for r in resource_set:
# if we overload dimensions with multiple resources we get
# the statistics/average over those resources.
dimensions = self.get_dimensions(r)
collected_metrics = r.setdefault('c7n.metrics', {})
# Note this annotation cache is policy scoped, not across
# policies, still the lack of full qualification on the key
# means multiple filters within a policy using the same metric
# across different periods or dimensions would be problematic.
key = "%s.%s.%s" % (self.namespace, self.metric, self.statistics)
if key not in collected_metrics:
collected_metrics[key] = client.get_metric_statistics(
Namespace=self.namespace,
MetricName=self.metric,
Statistics=[self.statistics],
StartTime=self.start,
EndTime=self.end,
Period=self.period,
Dimensions=dimensions)['Datapoints']
if len(collected_metrics[key]) == 0:
continue
if self.data.get('percent-attr'):
rvalue = r[self.data.get('percent-attr')]
if self.data.get('attr-multiplier'):
rvalue = rvalue * self.data['attr-multiplier']
percent = (collected_metrics[key][0][self.statistics] /
rvalue * 100)
if self.op(percent, self.value):
matched.append(r)
elif self.op(collected_metrics[key][0][self.statistics],
self.value):
matched.append(r)
return matched
class MetricsFilter(Filter):
"""Supports cloud watch metrics filters on resources.
All resources that have cloud watch metrics are supported.
Docs on cloud watch metrics
- GetMetricStatistics
https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_GetMetricStatistics.html
- Supported Metrics
https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/aws-services-cloudwatch-metrics.html
.. code-block:: yaml
- name: ec2-underutilized
resource: ec2
filters:
- type: metrics
name: CPUUtilization
days: 4
period: 86400
value: 30
op: less-than
Note periods when a resource is not sending metrics are not part
of calculated statistics as in the case of a stopped ec2 instance,
nor for resources to new to have existed the entire
period. ie. being stopped for an ec2 instance wouldn't lower the
average cpu utilization.
The "missing-value" key allows a policy to specify a default
value when CloudWatch has no data to report:
.. code-block:: yaml
- name: elb-low-request-count
resource: elb
filters:
- type: metrics
name: RequestCount
statistics: Sum
days: 7
value: 7
missing-value: 0
op: less-than
This policy matches any ELB with fewer than 7 requests for the past week.
ELBs with no requests during that time will have an empty set of metrics.
Rather than skipping those resources, "missing-value: 0" causes the
policy to treat their request counts as 0.
Note the default statistic for metrics is Average.
"""
schema = type_schema(
'metrics',
**{
'namespace': {
'type': 'string'
},
'name': {
'type': 'string'
},
'dimensions': {
'type': 'object',
'patternProperties': {
'^.*$': {
'type': 'string'
}
}
},
# Type choices
'statistics': {
'type': 'string',
'enum':
['Average', 'Sum', 'Maximum', 'Minimum', 'SampleCount']
},
'days': {
'type': 'number'
},
'op': {
'type': 'string',
'enum': list(OPERATORS.keys())
},
'value': {
'type': 'number'
},
'period': {
'type': 'number'
},
'attr-multiplier': {
'type': 'number'
},
'percent-attr': {
'type': 'string'
},
'missing-value': {
'type': 'number'
},
'required': ('value', 'name')
})
schema_alias = True
permissions = ("cloudwatch:GetMetricStatistics", )
MAX_QUERY_POINTS = 50850
MAX_RESULT_POINTS = 1440
# Default per service, for overloaded services like ec2
# we do type specific default namespace annotation
# specifically AWS/EBS and AWS/EC2Spot
# ditto for spot fleet
DEFAULT_NAMESPACE = {
'cloudfront': 'AWS/CloudFront',
'cloudsearch': 'AWS/CloudSearch',
'dynamodb': 'AWS/DynamoDB',
'ecs': 'AWS/ECS',
'efs': 'AWS/EFS',
'elasticache': 'AWS/ElastiCache',
'ec2': 'AWS/EC2',
'elb': 'AWS/ELB',
'elbv2': 'AWS/ApplicationELB',
'emr': 'AWS/ElasticMapReduce',
'es': 'AWS/ES',
'events': 'AWS/Events',
'firehose': 'AWS/Firehose',
'kinesis': 'AWS/Kinesis',
'lambda': 'AWS/Lambda',
'logs': 'AWS/Logs',
'redshift': 'AWS/Redshift',
'rds': 'AWS/RDS',
'route53': 'AWS/Route53',
's3': 'AWS/S3',
'sns': 'AWS/SNS',
'sqs': 'AWS/SQS',
'workspaces': 'AWS/WorkSpaces',
}
def process(self, resources, event=None):
days = self.data.get('days', 14)
duration = timedelta(days)
self.metric = self.data['name']
self.end = datetime.utcnow()
self.start = self.end - duration
self.period = int(self.data.get('period', duration.total_seconds()))
self.statistics = self.data.get('statistics', 'Average')
self.model = self.manager.get_model()
self.op = OPERATORS[self.data.get('op', 'less-than')]
self.value = self.data['value']
ns = self.data.get('namespace')
if not ns:
ns = getattr(self.model, 'metrics_namespace', None)
if not ns:
ns = self.DEFAULT_NAMESPACE[self.model.service]
self.namespace = ns
self.log.debug("Querying metrics for %d", len(resources))
matched = []
with self.executor_factory(max_workers=3) as w:
futures = []
for resource_set in chunks(resources, 50):
futures.append(
w.submit(self.process_resource_set, resource_set))
for f in as_completed(futures):
if f.exception():
self.log.warning("CW Retrieval error: %s" % f.exception())
continue
matched.extend(f.result())
return matched
def get_dimensions(self, resource):
return [{
'Name': self.model.dimension,
'Value': resource[self.model.dimension]
}]
def get_user_dimensions(self):
dims = []
if 'dimensions' not in self.data:
return dims
for k, v in self.data['dimensions'].items():
dims.append({'Name': k, 'Value': v})
return dims
def process_resource_set(self, resource_set):
client = local_session(
self.manager.session_factory).client('cloudwatch')
matched = []
for r in resource_set:
# if we overload dimensions with multiple resources we get
# the statistics/average over those resources.
dimensions = self.get_dimensions(r)
# Merge in any filter specified metrics, get_dimensions is
# commonly overridden so we can't do it there.
dimensions.extend(self.get_user_dimensions())
collected_metrics = r.setdefault('c7n.metrics', {})
# Note this annotation cache is policy scoped, not across
# policies, still the lack of full qualification on the key
# means multiple filters within a policy using the same metric
# across different periods or dimensions would be problematic.
key = "%s.%s.%s" % (self.namespace, self.metric, self.statistics)
if key not in collected_metrics:
collected_metrics[key] = client.get_metric_statistics(
Namespace=self.namespace,
MetricName=self.metric,
Statistics=[self.statistics],
StartTime=self.start,
EndTime=self.end,
Period=self.period,
Dimensions=dimensions)['Datapoints']
# In certain cases CloudWatch reports no data for a metric.
# If the policy specifies a fill value for missing data, add
# that here before testing for matches. Otherwise, skip
# matching entirely.
if len(collected_metrics[key]) == 0:
if 'missing-value' not in self.data:
continue
collected_metrics[key].append({
'Timestamp':
self.start,
self.statistics:
self.data['missing-value'],
'c7n:detail':
'Fill value for missing data'
})
if self.data.get('percent-attr'):
rvalue = r[self.data.get('percent-attr')]
if self.data.get('attr-multiplier'):
rvalue = rvalue * self.data['attr-multiplier']
percent = (collected_metrics[key][0][self.statistics] /
rvalue * 100)
if self.op(percent, self.value):
matched.append(r)
elif self.op(collected_metrics[key][0][self.statistics],
self.value):
matched.append(r)
return matched
def generate(resource_types=()):
resource_defs = {}
definitions = {
'resources': resource_defs,
'iam-statement': {
'additionalProperties':
False,
'type':
'object',
'properties': {
'Sid': {
'type': 'string'
},
'Effect': {
'type': 'string',
'enum': ['Allow', 'Deny']
},
'Principal': {
'anyOf': [{
'type': 'string'
}, {
'type': 'object'
}, {
'type': 'array'
}]
},
'NotPrincipal': {
'anyOf': [{
'type': 'object'
}, {
'type': 'array'
}]
},
'Action': {
'anyOf': [{
'type': 'string'
}, {
'type': 'array'
}]
},
'NotAction': {
'anyOf': [{
'type': 'string'
}, {
'type': 'array'
}]
},
'Resource': {
'anyOf': [{
'type': 'string'
}, {
'type': 'array'
}]
},
'NotResource': {
'anyOf': [{
'type': 'string'
}, {
'type': 'array'
}]
},
'Condition': {
'type': 'object'
}
},
'required': ['Sid', 'Effect'],
'oneOf': [{
'required': ['Principal', 'Action', 'Resource']
}, {
'required': ['NotPrincipal', 'Action', 'Resource']
}, {
'required': ['Principal', 'NotAction', 'Resource']
}, {
'required': ['NotPrincipal', 'NotAction', 'Resource']
}, {
'required': ['Principal', 'Action', 'NotResource']
}, {
'required': ['NotPrincipal', 'Action', 'NotResource']
}, {
'required': ['Principal', 'NotAction', 'NotResource']
}, {
'required': ['NotPrincipal', 'NotAction', 'NotResource']
}]
},
'actions': {},
'filters': {
'value': ValueFilter.schema,
'event': EventFilter.schema,
'age': AgeFilter.schema,
# Shortcut form of value filter as k=v
'valuekv': {
'type': 'object',
'minProperties': 1,
'maxProperties': 1
},
},
'filters_common': {
'comparison_operators': {
'enum': list(OPERATORS.keys())
},
'value_types': {
'enum': VALUE_TYPES
},
'value_from': ValuesFrom.schema,
'value': {
'oneOf': [{
'type': 'array'
}, {
'type': 'string'
}, {
'type': 'boolean'
}, {
'type': 'number'
}, {
'type': 'null'
}]
},
},
'policy': {
'type': 'object',
'required': ['name', 'resource'],
'additionalProperties': False,
'properties': {
'name': {
'type': 'string',
'pattern': "^[A-z][A-z0-9]*(-[A-z0-9]+)*$"
},
'region': {
'type': 'string'
},
'tz': {
'type': 'string'
},
'start': {
'format': 'date-time'
},
'end': {
'format': 'date-time'
},
'resource': {
'type': 'string'
},
'max-resources': {
'anyOf': [{
'type': 'integer',
'minimum': 1
}, {
'$ref': '#/definitions/max-resources-properties'
}]
},
'max-resources-percent': {
'type': 'number',
'minimum': 0,
'maximum': 100
},
'comment': {
'type': 'string'
},
'comments': {
'type': 'string'
},
'description': {
'type': 'string'
},
'tags': {
'type': 'array',
'items': {
'type': 'string'
}
},
'mode': {
'$ref': '#/definitions/policy-mode'
},
'source': {
'enum': ['describe', 'config']
},
'actions': {
'type': 'array',
},
'filters': {
'type': 'array'
},
#
# unclear if this should be allowed, it kills resource
# cache coherency between policies, and we need to
# generalize server side query mechanisms, currently
# this only for ec2 instance queries. limitations
# in json schema inheritance prevent us from doing this
# on a type specific basis
# https://stackoverflow.com/questions/22689900/json-schema-allof-with-additionalproperties
'query': {
'type': 'array',
'items': {
'type': 'object'
}
}
},
},
'policy-mode': {
'anyOf': [e.schema for _, e in execution.items()],
},
'max-resources-properties': {
'type': 'object',
'properties': {
'amount': {
"type": 'integer',
'minimum': 1
},
'op': {
'enum': ['or', 'and']
},
'percent': {
'type': 'number',
'minimum': 0,
'maximum': 100
}
}
}
}
resource_refs = []
for cloud_name, cloud_type in clouds.items():
for type_name, resource_type in cloud_type.resources.items():
if resource_types and type_name not in resource_types:
continue
alias_name = None
r_type_name = "%s.%s" % (cloud_name, type_name)
if cloud_name == 'aws':
alias_name = type_name
resource_refs.append(
process_resource(r_type_name, resource_type, resource_defs,
alias_name, definitions))
schema = {
"$schema": "http://json-schema.org/draft-07/schema#",
'id': 'http://schema.cloudcustodian.io/v0/custodian.json',
'definitions': definitions,
'type': 'object',
'required': ['policies'],
'additionalProperties': False,
'properties': {
'vars': {
'type': 'object'
},
'policies': {
'type': 'array',
'additionalItems': False,
'items': {
'anyOf': resource_refs
}
}
}
}
return schema
def generate(resource_types=()):
resource_defs = {}
definitions = {
'resources': resource_defs,
'iam-statement': {
'additionalProperties': False,
'type': 'object',
'properties': {
'Sid': {'type': 'string'},
'Effect': {'type': 'string', 'enum': ['Allow', 'Deny']},
'Principal': {'anyOf': [
{'type': 'string'},
{'type': 'object'}, {'type': 'array'}]},
'NotPrincipal': {'anyOf': [{'type': 'object'}, {'type': 'array'}]},
'Action': {'anyOf': [{'type': 'string'}, {'type': 'array'}]},
'NotAction': {'anyOf': [{'type': 'string'}, {'type': 'array'}]},
'Resource': {'anyOf': [{'type': 'string'}, {'type': 'array'}]},
'NotResource': {'anyOf': [{'type': 'string'}, {'type': 'array'}]},
'Condition': {'type': 'object'}
},
'required': ['Sid', 'Effect'],
'oneOf': [
{'required': ['Principal', 'Action', 'Resource']},
{'required': ['NotPrincipal', 'Action', 'Resource']},
{'required': ['Principal', 'NotAction', 'Resource']},
{'required': ['NotPrincipal', 'NotAction', 'Resource']},
{'required': ['Principal', 'Action', 'NotResource']},
{'required': ['NotPrincipal', 'Action', 'NotResource']},
{'required': ['Principal', 'NotAction', 'NotResource']},
{'required': ['NotPrincipal', 'NotAction', 'NotResource']}
]
},
'actions': {},
'filters': {
'value': ValueFilter.schema,
'event': EventFilter.schema,
'age': AgeFilter.schema,
# Shortcut form of value filter as k=v
'valuekv': {
'type': 'object',
'minProperties': 1,
'maxProperties': 1},
},
'filters_common': {
'comparison_operators': {
'enum': list(OPERATORS.keys())},
'value_types': {'enum': VALUE_TYPES},
'value_from': ValuesFrom.schema,
'value': {'oneOf': [
{'type': 'array'},
{'type': 'string'},
{'type': 'boolean'},
{'type': 'number'},
{'type': 'null'}]},
},
'policy': {
'type': 'object',
'required': ['name', 'resource'],
'additionalProperties': False,
'properties': {
'name': {
'type': 'string',
'pattern': "^[A-z][A-z0-9]*(-[A-z0-9]+)*$"},
'region': {'type': 'string'},
'tz': {'type': 'string'},
'start': {'format': 'date-time'},
'end': {'format': 'date-time'},
'resource': {'type': 'string'},
'max-resources': {'anyOf': [
{'type': 'integer', 'minimum': 1},
{'$ref': '#/definitions/max-resources-properties'}
]},
'max-resources-percent': {'type': 'number', 'minimum': 0, 'maximum': 100},
'comment': {'type': 'string'},
'comments': {'type': 'string'},
'description': {'type': 'string'},
'tags': {'type': 'array', 'items': {'type': 'string'}},
'mode': {'$ref': '#/definitions/policy-mode'},
'source': {'enum': ['describe', 'config']},
'actions': {
'type': 'array',
},
'filters': {
'type': 'array'
},
#
# unclear if this should be allowed, it kills resource
# cache coherency between policies, and we need to
# generalize server side query mechanisms, currently
# this only for ec2 instance queries. limitations
# in json schema inheritance prevent us from doing this
# on a type specific basis
# https://stackoverflow.com/questions/22689900/json-schema-allof-with-additionalproperties
'query': {
'type': 'array', 'items': {'type': 'object'}}
},
},
'policy-mode': {
'anyOf': [e.schema for _, e in execution.items()],
},
'max-resources-properties': {
'type': 'object',
'properties': {
'amount': {"type": 'integer', 'minimum': 1},
'op': {'enum': ['or', 'and']},
'percent': {'type': 'number', 'minimum': 0, 'maximum': 100}
}
}
}
resource_refs = []
for cloud_name, cloud_type in clouds.items():
for type_name, resource_type in cloud_type.resources.items():
if resource_types and type_name not in resource_types:
continue
alias_name = None
r_type_name = "%s.%s" % (cloud_name, type_name)
if cloud_name == 'aws':
alias_name = type_name
resource_refs.append(
process_resource(
r_type_name,
resource_type,
resource_defs,
alias_name,
definitions
))
schema = {
"$schema": "http://json-schema.org/draft-07/schema#",
'id': 'http://schema.cloudcustodian.io/v0/custodian.json',
'definitions': definitions,
'type': 'object',
'required': ['policies'],
'additionalProperties': False,
'properties': {
'vars': {'type': 'object'},
'policies': {
'type': 'array',
'additionalItems': False,
'items': {'anyOf': resource_refs}
}
}
}
return schema
