def lambda_handler(event, context):
"""
Ingest CloudWatch Metric statistics to Humio repository.
:param event: Event data.
:type event: dict
:param context: Lambda context object.
:type context: obj
:return: None
"""
# Persist variables across lambda invocations.
if not _is_setup:
helpers.setup()
# Load user defined configurations for the API request.
configurations = json.load(
open("conf_metric_statistics_ingester.json", "r"))
# Make CloudWatch:GetMetricStatistics API request.
metric_statistics, api_parameters = get_metric_statistics(configurations)
# Used for debugging.
logger.debug("Statistics from CloudWatch Metrics: %s" % metric_statistics)
# Format metric data to Humio event data.
humio_events = create_humio_events(metric_statistics, api_parameters)
# Send Humio event data to Humio.
request = helpers.ingest_events(humio_events, "cloudwatch_metrics")
# Debug the response.
response = request.text
logger.debug("Got response %s from Humio." % response)
def lambda_handler(event, context):
"""
Ingest CloudWatch Metrics data to Humio repository.
:param event: Event data.
:type event: dict
:param context: Lambda context object.
:type context: obj
:return: None
"""
# Persist variables across lambda invocations.
if not _is_setup:
helpers.setup()
# Load user defined configurations for the API request.
configurations = json.load(open("conf_metric_ingester.json", "r"))
# Set next token if one is present in the event.
if "NextToken" in event.keys():
configurations["NextToken"] = event["NextToken"]
# Set default start time if none is present.
if "StartTime" not in configurations.keys():
if "StartTime" in event.keys():
configurations["StartTime"] = event["StartTime"]
else:
configurations["StartTime"] = (datetime.utcnow() - timedelta(minutes=15))\
.replace(tzinfo=timezone.utc).isoformat() # 15 minutes ago.
# Set default end time if none is present.
if "EndTime" not in configurations.keys():
if "EndTime" in event.keys():
configurations["EndTime"] = event["EndTime"]
else:
configurations["EndTime"] = datetime.utcnow()\
.replace(tzinfo=timezone.utc).isoformat() # Now.
# Make CloudWatch:GetMetricData API request.
metric_data = get_metric_data(configurations)
# If there is a next token in the metric data,
# then use this to retrieve the rest of the metrics recursively.
if "NextToken" in metric_data:
lambda_client = boto3.client("lambda")
# Pass on next token, start time, and end time.
event["NextToken"] = metric_data["NextToken"]
event["StartTime"] = configurations["StartTime"]
event["EndTime"] = configurations["EndTime"]
lambda_client.invoke(FunctionName=context.function_name,
InvocationType="Event",
Payload=json.dumps(event))
# Format metric data to Humio event data.
humio_events = create_humio_events(metric_data, configurations)
# Send Humio event data to Humio.
request = helpers.ingest_events(humio_events, "cloudwatch_metrics")
# Debug the response.
response = request.text
print("Got response %s from Humio." % response)
def lambda_handler(event, context):
"""
Extract log data from CloudWatch Logs events and
pass the data onto the Humio ingester.
:param event: Event data from CloudWatch Logs.
:type event: dict
:param context: Lambda context object.
:type context: obj
:return: None
"""
# Persist variables across lambda invocations.
if not _is_setup:
helpers.setup()
# Decode and unzip the log data.
decoded_event = helpers.decode_event(event)
# Debug output.
logger.debug("Event from CloudWatch Logs: %s" %
(json.dumps(decoded_event)))
# Extract the general attributes from the event batch.
batch_attrs = {
"owner":
decoded_event.get("owner", "undefined"),
"logGroup":
decoded_event.get("logGroup", "undefined"),
"logStream":
decoded_event.get("logStream", "undefined"),
"messageType":
decoded_event.get("messageType", "undefined"),
"subscriptionFilters":
decoded_event.get("subscriptionFilters", "undefined"),
}
# Parse out the service name.
log_group_parser = re.compile("^/aws/(lambda|apigateway)/(.*)")
parsed_log_group = log_group_parser.match(decoded_event.get("", ""))
if parsed_log_group:
batch_attrs.update({
"awsServiceName": parsed_log_group.group(1),
"parsedLogGroupName": parsed_log_group.group(2)
})
# Flatten the events from CloudWatch Logs.
humio_events = []
for log_event in decoded_event["logEvents"]:
message = log_event["message"]
# Create the attributes.
attributes = {}
attributes.update(batch_attrs)
attributes.update(helpers.parse_message(message))
# Append the flattened event
humio_events.append({
"timestamp": log_event["timestamp"],
"rawstring": message,
"kvparse": True,
"attributes": attributes,
})
# Make request to Humio.
request = helpers.ingest_events(humio_events, 'cloudwatch_logs')
response = request.text
# Debug output.
logger.debug("Got response %s from Humio." % response)