def _index_data(self, records):
indexed = 0
events = []
writer = self._config[tac.event_writer]
for rec in records:
if not rec["Data"]:
continue
indexed += 1
handled = self._handle_fmt_record(rec, events)
if not handled:
evt_time = tacommon.total_seconds(
rec["ApproximateArrivalTimestamp"])
data = scutil.try_convert_to_json(rec["Data"])
event = writer.create_event(
index=self._config.get(tac.index, "default"),
host=self._config.get(tac.host, ""),
source=self._source,
sourcetype=self._config.get(tac.sourcetype, "aws:kinesis"),
time=evt_time,
unbroken=False,
done=False,
events=data)
events.append(event)
try:
writer.write_events(events, retry=30)
except Exception:
logger.error("Failed to index events %s, error=%s", self._msg,
traceback.format_exc())
raise
logger.debug("Indexed %d records %s", len(events), self._msg)
return indexed
def _index_compliance_summary(self, rule_name):
writer = self._config[tac.event_writer]
sourcetype = self._config.get(tac.sourcetype, "aws:config:rule")
response = self._client.get_compliance_summary_by_config_rule(
ConfigRuleName=rule_name)
if not tacommon.is_http_ok(response):
logger.error(
"Failed to collect compliance summary for "
"rule=%s, errorcode=%s", rule_name,
tacommon.http_code(response))
return
summary = response.get("ComplianceSummary")
if not summary:
return
evt_time = tacommon.total_seconds(
summary["ComplianceSummaryTimestamp"])
summary["ComplianceSummaryTimestamp"] = "{}".format(
summary["ComplianceSummaryTimestamp"])
summary["ConfigRuleName"] = rule_name
event = writer.create_event(index=self._config.get(
tac.index, "default"),
host=self._config.get(tac.host, ""),
source=self._source_summary,
sourcetype=sourcetype,
time=evt_time,
unbroken=False,
done=False,
events=summary)
writer.write_events((event, ))
def _last_evaluation_times(self, rule_name=""):
writer = self._config[tac.event_writer]
response = self._client.describe_config_rule_evaluation_status(
ConfigRuleNames=[rule_name])
if not tacommon.is_http_ok(response):
logger.error(
"Failed to describe config rule evaluation status, "
"errorcode=%s", tacommon.http_code(response))
return None
statuses = response.get("ConfigRulesEvaluationStatus")
if not statuses:
return None
dkeys = [
"LastSuccessfulInvocationTime", "LastFailedInvocationTime",
"LastSuccessfulEvaluationTime", "LastFailedEvaluationTime",
"FirstActivatedTime"
]
sourcetype = self._config.get(tac.sourcetype, "aws:config:rule")
last_times, events = [], []
for status in statuses:
if status.get("LastSuccessfulEvaluationTime"):
evt_time = tacommon.total_seconds(
status["LastSuccessfulEvaluationTime"])
else:
evt_time = ""
ckpt_time = self._ckpt.last_evaluation_time(
self._config[tac.region], self._config[tac.datainput],
status["ConfigRuleName"])
last_times.append((evt_time, ckpt_time))
if ckpt_time == evt_time:
continue
for key in dkeys:
if key in status:
status[key] = "{}".format(status[key])
event = writer.create_event(
index=self._config.get(tac.index, "default"),
host=self._config.get(tac.host, ""),
source=self._source_status,
sourcetype=sourcetype,
time=evt_time,
unbroken=False,
done=False,
events=status,
)
events.append(event)
if events:
writer.write_events(events)
return last_times
def _index_run(self, data):
etime = tacommon.total_seconds(data['completedAt'])
event = self._writer.create_event(
index=self._config.get(tac.index, 'default'),
host=self._config.get(tac.host, ''),
source=self._source,
sourcetype=self._source_type,
time=etime,
unbroken=False,
done=False,
events=data,
)
self._writer.write_events((event, ))
def _index_findings(self, findings):
for item in findings:
etime = tacommon.total_seconds(item['updatedAt'])
event = self._writer.create_event(
index=self._config.get(tac.index, "default"),
host=self._config.get(tac.host, ""),
source=self._source,
sourcetype=self._source_type,
time=etime,
unbroken=False,
done=False,
events=item,
)
self._writer.write_events((event, ))
def _index_compliance_details(self, rule_name):
sourcetype = self._config.get(tac.sourcetype, "aws:config:rule")
writer = self._config[tac.event_writer]
next_token = ""
while 1:
response = self._client.get_compliance_details_by_config_rule(
ConfigRuleName=rule_name, NextToken=next_token)
if not tacommon.is_http_ok(response):
logger.error(
"Failed to collect compliance details for "
"rule=%s, errorcode=%s", rule_name,
tacommon.http_code(response))
return
compliances = response.get("EvaluationResults")
if not compliances:
return
events = []
for compliance in compliances:
evt_time = compliance["ResultRecordedTime"]
compliance["ResultRecordedTime"] = "{}".format(evt_time)
compliance["ConfigRuleInvokedTime"] = "{}".format(
compliance["ConfigRuleInvokedTime"])
compliance["EvaluationResultIdentifier"][
"OrderingTimestamp"] = "{}".format(
compliance["EvaluationResultIdentifier"]
["OrderingTimestamp"])
evt_time = tacommon.total_seconds(evt_time)
event = writer.create_event(
index=self._config.get(tac.index, "default"),
host=self._config.get(tac.host, ""),
source=self._source_detail,
sourcetype=sourcetype,
time=evt_time,
unbroken=False,
done=False,
events=compliance)
events.append(event)
writer.write_events(events)
next_token = response.get("NextToken")
if not next_token:
return
def _index_data(self, records):
writer = self._config[tac.event_writer]
sourcetype = self._config.get(tac.sourcetype, "aws:cloudwatch")
events = []
for dimension, datapoints in records:
for datapoint in datapoints:
datapoint.update(self._supplemental_data)
datapoint[acc.metric_dimensions] = self._build_dimension_str(
dimension)
datapoint[acc.metric_name] = dimension["MetricName"]
evt_time = tacommon.total_seconds(datapoint["Timestamp"])
del datapoint["Timestamp"]
event = writer.create_event(
index=self._config.get(tac.index, "default"),
host=self._config.get(tac.host, ""),
source=self._source,
sourcetype=sourcetype,
time=evt_time,
unbroken=False,
done=False,
events=datapoint)
events.append(event)
writer.write_events(events, retry=10)
def _index_data(self, records):
writer = self._config[tac.event_writer]
sourcetype = self._config.get(tac.sourcetype, "aws:cloudwatch")
events = []
use_metric_format = self._config[acc.use_metric_format]
if use_metric_format:
client = self._client.require_boto3_ec2_client()
self._instances_data_list = self._get_instance_metadata(client)
for dimension, datapoints in records:
for datapoint in datapoints:
datapoint.update(self._supplemental_data)
evt_time = tacommon.total_seconds(datapoint["Timestamp"])
del datapoint["Timestamp"]
if use_metric_format:
metric_datapoint = self._create_metric_datapoint(
dimension, datapoint)
data = self._build_metric_str(metric_datapoint, evt_time)
fields = metric_datapoint
else:
datapoint[
acc.metric_dimensions] = self._build_dimension_str(
dimension)
datapoint[acc.metric_name] = dimension["MetricName"]
data = datapoint
fields = None
event = writer.create_event(
index=self._config.get(tac.index, "default"),
host=self._config.get(tac.host, ""),
source=self._source,
sourcetype=sourcetype,
time=evt_time,
unbroken=False,
done=False,
events=data,
fields=fields)
events.append(event)
writer.write_events(events, retry=10)