def jmes_path(source_data: celtypes.Value,
path_source: celtypes.StringType) -> celtypes.Value:
"""
Apply JMESPath to an object read from from a URL.
"""
expression = jmespath.compile(path_source)
return json_to_cel(expression.search(source_data))
def jmes_path_map(source_data: celtypes.ListType,
path_source: celtypes.StringType) -> celtypes.ListType:
"""
Apply JMESPath to a each object read from from a URL.
This is for ndjson, nljson and jsonl files.
"""
expression = jmespath.compile(path_source)
return celtypes.ListType(
[json_to_cel(expression.search(row)) for row in source_data])
def get_metrics(resource: celtypes.MapType,
request: celtypes.MapType) -> celtypes.Value:
"""
Reach into C7N and make a statistics request using the current C7N filter.
This uses the module-global ``C7N`` namespace to access the original filter and policy.
This builds a request object that is passed through to AWS via the :func:`get_raw_metrics`
function.
The ``request`` parameter is a Mapping with the following keys and values:
::
Resource.get_metrics({"MetricName": "CPUUtilization", "Statistic": "Average",
"StartTime": Now - duration("4d"), "EndTime": Now, "Period": duration("86400s")}
).exists(m, m < 30)
The namespace is derived from the ``C7N.policy``. The dimensions are derived from
the ``C7N.fiter.model``.
.. todo:: Refactor C7N
Provide a :py:class:`MetricsAccess` mixin in a :py:class:`CELFilter` class.
We want to have the metrics processing in the new :py:class:`CELFilter` instance.
"""
dimension = celtypes.StringType(C7N.filter.manager.get_model().dimension)
namespace = celtypes.StringType(C7N.filter.manager.resource_type)
# TODO: Varies by resource/policy type. Each policy's model may have different dimensions.
dimensions = json_to_cel([{
'Name': dimension,
'Value': resource.get(dimension)
}])
raw_metrics = cast(
celtypes.ListType,
get_raw_metrics(
celtypes.MapType({
celtypes.StringType("Namespace"):
namespace,
celtypes.StringType("MetricName"):
request["MetricName"],
celtypes.StringType("Dimensions"):
dimensions,
celtypes.StringType("Statistics"): [request["Statistic"]],
celtypes.StringType("StartTime"):
request["StartTime"],
celtypes.StringType("EndTime"):
request["EndTime"],
celtypes.StringType("Period"):
request["Period"],
})))
return celtypes.ListType([
cast(celtypes.MapType, item).get(request["Statistic"])
for item in raw_metrics
])
def flow_logs(resource: celtypes.MapType, ) -> celtypes.Value:
"""
Reach into C7N and make a get_related() request using the current C7N filter.
.. todo:: Refactor C7N
Provide a separate function to get the flow logs, separate from the
the filter processing.
"""
client = C7N.filter.client('ec2')
logs = client.describe_flow_logs().get('FlowLogs', ())
m = C7N.filter.manager.get_model()
resource_map: Dict[str, List[Dict[str, Any]]] = {}
for fl in logs:
resource_map.setdefault(fl['ResourceId'], []).append(fl)
if resource.get(m.id) in resource_map:
flogs = resource_map[cast(str, resource.get(m.id))]
return json_to_cel(flogs)
return json_to_cel([])
def get_related_ids(resource: celtypes.MapType, ) -> celtypes.Value:
"""
Reach into C7N and make a get_related_ids() request using the current C7N filter.
.. todo:: Refactor C7N
Provide the :py:class:`RelatedResourceFilter` mixin in a :py:class:`CELFilter` class.
We want to have the related id's details in the new :py:class:`CELFilter` instance.
"""
# Assuming the :py:class:`CELFilter` class has this method extracted from the legacy filter.
security_group_ids = C7N.filter.get_related_ids(resource)
return json_to_cel(security_group_ids)
def subnet(subnet_id: celtypes.Value, ) -> celtypes.Value:
"""
Reach into C7N and make a get_related() request using the current C7N filter to get
the subnet.
.. todo:: Refactor C7N
Provide the :py:class:`RelatedResourceFilter` mixin in a :py:class:`CELFilter` class.
We want to have the related id's details in the new :py:class:`CELFilter` instance.
See :py:class:`VpcSubnetFilter` subclass of :py:class:`RelatedResourceFilter`.
"""
# Get related ID's first, then get items for the related ID's.
subnets = C7N.filter.get_related([subnet_id])
return json_to_cel(subnets)
def security_group(security_group_id: celtypes.Value, ) -> celtypes.Value:
"""
Reach into C7N and make a get_related() request using the current C7N filter to get
the security group.
.. todo:: Refactor C7N
Provide the :py:class:`RelatedResourceFilter` mixin in a :py:class:`CELFilter` class.
We want to have the related id's details in the new :py:class:`CELFilter` instance.
See :py:class:`VpcSecurityGroupFilter` subclass of :py:class:`RelatedResourceFilter`.
"""
# Assuming the :py:class:`CELFilter` class has this method extracted from the legacy filter.
security_groups = C7N.filter.get_related([security_group_id])
return json_to_cel(security_groups)
def parse_text(source_text: celtypes.StringType,
format: celtypes.StringType) -> celtypes.Value:
"""
Parse raw text using a given format.
"""
if format == "json":
return json_to_cel(json.loads(source_text))
elif format == "txt":
return celtypes.ListType([
celtypes.StringType(s.rstrip()) for s in source_text.splitlines()
])
elif format in ("ldjson", "ndjson", "jsonl"):
return celtypes.ListType(
[json_to_cel(json.loads(s)) for s in source_text.splitlines()])
elif format == "csv":
return celtypes.ListType(
[json_to_cel(row) for row in csv.reader(io.StringIO(source_text))])
elif format == "csv2dict":
return celtypes.ListType([
json_to_cel(row)
for row in csv.DictReader(io.StringIO(source_text))
])
else:
raise ValueError(f"Unsupported format: {format!r}") # pragma: no cover
def get_raw_metrics(request: celtypes.MapType) -> celtypes.Value:
"""
Reach into C7N and make a statistics request using the current C7N filter object.
This uses the module-global ``C7N`` namespace to access the original filter's manager.
The ``request`` parameter is the request object that is passed through to AWS via
the current C7N filter's manager. The request is a Mapping with the following keys and values:
::
get_raw_metrics({
"Namespace": "AWS/EC2",
"MetricName": "CPUUtilization",
"Dimensions": {"Name": "InstanceId", "Value": Resource.InstanceId},
"Statistics": ["Average"],
"StartTime": Now - duration("4d"),
"EndTime": Now,
"Period": duration("86400s")
})
The request is passed through to AWS more-or-less directly. The result is a CEL
list of values for then requested statistic. A ``.map()`` macro
can be used to compute additional details. An ``.exists()`` macro can filter the
data to look for actionable values.
Generally, C7N requests in bunches of 50 per client connection.
A worker pool processes the batches to keep from overwhelming AWS with
metrics requests.
See :py:class:`c7n.filters.metrics.MetricsFilter`. This filter collects
metrics and applies the filter decision to items in each batch.
The :py:meth:`process` and :py:meth:`process_resource_set` methods
need to be refactored into several pieces:
- :py:meth:`process_resource_set`. This is the existing interface.
This calls :py:meth:`prepare_query` to create the various query
parameters. It then creates a worker pool and applies :py:meth:`process_resource_set`
to chunks of 50 resources.
- :py:meth:`prepare_query`. This is new. It prepares the parameters
for :py:meth:`client.get_metric_statistics`.
- :py:meth:`process_resource_set`. This is the existing interface.
It gets a client and then calls :py:meth:`get_resource_statistics` with the client
and each resource. It calls :py:meth:`filter_resource_statistics` on the results
of :py:meth:`client.get_metric_statistics`.
- :py:meth:`get_resource_statistics`. Given a client and a resource,
this function will set the resource's ``"c7n.metrics"`` attribute with current
statistics. This is the ``['Datapoints']`` value. It returns the [self.statistics]
item from each dictionary in the metrics list of dictionaries.
- :py:meth:`filter_resource_statistics`. Given a resource, this function will apply the
missing-value, the percent-attr and attr-multiplier transformations to the
resource's ``"c7n.metrics"``.
It will apply the filter op and value. All of these things better represented in CEL.
We need to be able to use code something like this:
::
C7N.filter.prepare_query(C7N.policy.resources)
data = C7N.filter.get_resource_statistics(client, resource)
return json_to_cel(data)
.. todo:: Refactor C7N
Provide a :py:class:`MetricsAccess` mixin in a :py:class:`CELFilter` class.
We want to have the metrics processing in the new :py:class:`CELFilter` instance.
"""
# The preferred design reaches into the policy for an access strategy to get details.
# data = C7N.filter.get_resource_statistics(
# Namespace = request["Namespace"],
# MetricName = request["MetricName"],
# Statistics = [request["Statistics"]],
# StartTime = request["StartTime"],
# EndTime = request["EndTime"],
# Period = request["Period"],
# Dimensions = request["Dimensions"],
# )
# An alternative design which acquires data outside the filter object's cache
client = C7N.filter.manager.session_factory().client('cloudwatch')
print(f"Client {client}")
data = client.get_metric_statistics(
Namespace=request["Namespace"],
MetricName=request["MetricName"],
Statistics=[request["Statistics"]],
StartTime=request["StartTime"],
EndTime=request["EndTime"],
Period=request["Period"],
Dimensions=request["Dimensions"],
)['Datapoints']
print(f"data {data}")
return json_to_cel(data)