def test_json_to_cel():
"""GIVEN JSON doc; WHEN json_to_cell(); THEN expected conversions applied."""
doc = [
{
"bool": True
},
{
"numbers": [2.71828, 42]
},
{
"null": None
},
{
"string": 'embedded "quote"'
},
]
actual = celpy.json_to_cel(doc)
expected = celtypes.ListType([
celtypes.MapType(
{celtypes.StringType("bool"): celtypes.BoolType(True)}),
celtypes.MapType({
celtypes.StringType("numbers"):
celtypes.ListType(
[celtypes.DoubleType(2.71828),
celtypes.IntType(42)])
}),
celtypes.MapType({celtypes.StringType("null"): None}),
celtypes.MapType({
celtypes.StringType("string"):
celtypes.StringType('embedded "quote"')
}),
])
assert actual == expected
def json_to_cel(document: JSON) -> celtypes.Value:
"""Convert parsed JSON object from Python to CEL to the extent possible.
It's difficult to distinguish strings which should be timestamps or durations.
::
>>> from pprint import pprint
>>> from celpy.adapter import json_to_cel
>>> doc = json.loads('["str", 42, 3.14, null, true, {"hello": "world"}]')
>>> cel = json_to_cel(doc)
>>> pprint(cel)
ListType([StringType('str'), IntType(42), DoubleType(3.14), None, BoolType(True), \
MapType({StringType('hello'): StringType('world')})])
"""
if isinstance(document, bool):
return celtypes.BoolType(document)
elif isinstance(document, float):
return celtypes.DoubleType(document)
elif isinstance(document, int):
return celtypes.IntType(document)
elif isinstance(document, str):
return celtypes.StringType(document)
elif document is None:
return None
elif isinstance(document, List):
return celtypes.ListType([json_to_cel(item) for item in document])
elif isinstance(document, Dict):
return celtypes.MapType({
json_to_cel(key): json_to_cel(value)
for key, value in document.items()
})
else:
raise ValueError(
f"unexpected type {type(document)} in JSON structure {document!r}")
def jmes_path_map(source_data: celtypes.ListType,
path_source: celtypes.StringType) -> celtypes.ListType:
"""
Apply JMESPath to a each object read from from a URL.
This is for ndjson, nljson and jsonl files.
"""
expression = jmespath.compile(path_source)
return celtypes.ListType(
[json_to_cel(expression.search(row)) for row in source_data])
def get_metrics(resource: celtypes.MapType,
request: celtypes.MapType) -> celtypes.Value:
"""
Reach into C7N and make a statistics request using the current C7N filter.
This uses the module-global ``C7N`` namespace to access the original filter and policy.
This builds a request object that is passed through to AWS via the :func:`get_raw_metrics`
function.
The ``request`` parameter is a Mapping with the following keys and values:
::
Resource.get_metrics({"MetricName": "CPUUtilization", "Statistic": "Average",
"StartTime": Now - duration("4d"), "EndTime": Now, "Period": duration("86400s")}
).exists(m, m < 30)
The namespace is derived from the ``C7N.policy``. The dimensions are derived from
the ``C7N.fiter.model``.
.. todo:: Refactor C7N
Provide a :py:class:`MetricsAccess` mixin in a :py:class:`CELFilter` class.
We want to have the metrics processing in the new :py:class:`CELFilter` instance.
"""
dimension = celtypes.StringType(C7N.filter.manager.get_model().dimension)
namespace = celtypes.StringType(C7N.filter.manager.resource_type)
# TODO: Varies by resource/policy type. Each policy's model may have different dimensions.
dimensions = json_to_cel([{
'Name': dimension,
'Value': resource.get(dimension)
}])
raw_metrics = cast(
celtypes.ListType,
get_raw_metrics(
celtypes.MapType({
celtypes.StringType("Namespace"):
namespace,
celtypes.StringType("MetricName"):
request["MetricName"],
celtypes.StringType("Dimensions"):
dimensions,
celtypes.StringType("Statistics"): [request["Statistic"]],
celtypes.StringType("StartTime"):
request["StartTime"],
celtypes.StringType("EndTime"):
request["EndTime"],
celtypes.StringType("Period"):
request["Period"],
})))
return celtypes.ListType([
cast(celtypes.MapType, item).get(request["Statistic"])
for item in raw_metrics
])
def parse_text(source_text: celtypes.StringType,
format: celtypes.StringType) -> celtypes.Value:
"""
Parse raw text using a given format.
"""
if format == "json":
return json_to_cel(json.loads(source_text))
elif format == "txt":
return celtypes.ListType([
celtypes.StringType(s.rstrip()) for s in source_text.splitlines()
])
elif format in ("ldjson", "ndjson", "jsonl"):
return celtypes.ListType(
[json_to_cel(json.loads(s)) for s in source_text.splitlines()])
elif format == "csv":
return celtypes.ListType(
[json_to_cel(row) for row in csv.reader(io.StringIO(source_text))])
elif format == "csv2dict":
return celtypes.ListType([
json_to_cel(row)
for row in csv.DictReader(io.StringIO(source_text))
])
else:
raise ValueError(f"Unsupported format: {format!r}") # pragma: no cover
def test_decoder():
json_text = (
'{"bool": 1, "numbers": [2.71828, 42], "null": null, '
'"string": "embedded \\"quote\\"", "bytes": "Ynl0ZXM=", '
'"timestamp": "2009-02-13T23:31:30Z", "duration": "42s"}'
)
cel_obj = json.loads(json_text, cls=celpy.CELJSONDecoder)
assert cel_obj == celtypes.MapType({
celtypes.StringType('bool'): celtypes.IntType(1),
celtypes.StringType('bytes'): celtypes.StringType('Ynl0ZXM='),
celtypes.StringType('duration'): celtypes.StringType('42s'),
celtypes.StringType('null'): None,
celtypes.StringType('numbers'):
celtypes.ListType([celtypes.DoubleType(2.71828), celtypes.IntType(42)]),
celtypes.StringType('string'): celtypes.StringType('embedded "quote"'),
celtypes.StringType('timestamp'): celtypes.StringType('2009-02-13T23:31:30Z'),
})
def test_encoder():
cel_obj = celtypes.MapType(
{
celtypes.StringType("bool"): celtypes.BoolType(True),
celtypes.StringType("numbers"):
celtypes.ListType([
celtypes.DoubleType(2.71828), celtypes.UintType(42)
]),
celtypes.StringType("null"): None,
celtypes.StringType("string"): celtypes.StringType('embedded "quote"'),
celtypes.StringType("bytes"):
celtypes.BytesType(bytes([0x62, 0x79, 0x74, 0x65, 0x73])),
celtypes.StringType("timestamp"): celtypes.TimestampType('2009-02-13T23:31:30Z'),
celtypes.StringType("duration"): celtypes.DurationType('42s'),
}
)
json_text = json.dumps(cel_obj, cls=celpy.CELJSONEncoder)
assert (
json_text == '{"bool": true, "numbers": [2.71828, 42], "null": null, '
'"string": "embedded \\"quote\\"", "bytes": "Ynl0ZXM=", '
'"timestamp": "2009-02-13T23:31:30Z", "duration": "42s"}'
)
from celpy.celparser import CELParseError
from celpy import celtypes
logger = logging.getLogger("celpy")
# For argument parsing purposes.
# Note the reliance on `ast.literal_eval` for ListType and MapType conversions.
# Other types convert strings directly. These types need some help.
CLI_ARG_TYPES: Dict[str, celtypes.CELType] = {
"int": celtypes.IntType,
"uint": celtypes.UintType,
"double": celtypes.DoubleType,
"bool": celtypes.BoolType,
"string": celtypes.StringType,
"bytes": celtypes.BytesType,
"list": lambda arg: celtypes.ListType(ast.literal_eval(arg)),
"map": lambda arg: celtypes.MapType(ast.literal_eval(arg)),
"null_type": cast(Callable[[str], celtypes.Value], lambda x: None),
"single_duration": celtypes.DurationType,
"single_timestamp": celtypes.TimestampType,
"int64_value": celtypes.IntType,
"uint64_value": celtypes.UintType,
"double_value": celtypes.DoubleType,
"bool_value": celtypes.BoolType,
"string_value": celtypes.StringType,
"bytes_value": celtypes.BytesType,
"number_value": celtypes.DoubleType, # Ambiguous; can somtimes be integer.
"null_value": cast(Callable[[str], celtypes.Value], lambda x: None),
}