def _index_data(self, records): indexed = 0 events = [] writer = self._config[tac.event_writer] for rec in records: if not rec["Data"]: continue indexed += 1 handled = self._handle_fmt_record(rec, events) if not handled: evt_time = tacommon.total_seconds( rec["ApproximateArrivalTimestamp"]) data = scutil.try_convert_to_json(rec["Data"]) event = writer.create_event( index=self._config.get(tac.index, "default"), host=self._config.get(tac.host, ""), source=self._source, sourcetype=self._config.get(tac.sourcetype, "aws:kinesis"), time=evt_time, unbroken=False, done=False, events=data) events.append(event) try: writer.write_events(events, retry=30) except Exception: logger.error("Failed to index events %s, error=%s", self._msg, traceback.format_exc()) raise logger.debug("Indexed %d records %s", len(events), self._msg) return indexed
def _index_compliance_summary(self, rule_name): writer = self._config[tac.event_writer] sourcetype = self._config.get(tac.sourcetype, "aws:config:rule") response = self._client.get_compliance_summary_by_config_rule( ConfigRuleName=rule_name) if not tacommon.is_http_ok(response): logger.error( "Failed to collect compliance summary for " "rule=%s, errorcode=%s", rule_name, tacommon.http_code(response)) return summary = response.get("ComplianceSummary") if not summary: return evt_time = tacommon.total_seconds( summary["ComplianceSummaryTimestamp"]) summary["ComplianceSummaryTimestamp"] = "{}".format( summary["ComplianceSummaryTimestamp"]) summary["ConfigRuleName"] = rule_name event = writer.create_event(index=self._config.get( tac.index, "default"), host=self._config.get(tac.host, ""), source=self._source_summary, sourcetype=sourcetype, time=evt_time, unbroken=False, done=False, events=summary) writer.write_events((event, ))
def _last_evaluation_times(self, rule_name=""): writer = self._config[tac.event_writer] response = self._client.describe_config_rule_evaluation_status( ConfigRuleNames=[rule_name]) if not tacommon.is_http_ok(response): logger.error( "Failed to describe config rule evaluation status, " "errorcode=%s", tacommon.http_code(response)) return None statuses = response.get("ConfigRulesEvaluationStatus") if not statuses: return None dkeys = [ "LastSuccessfulInvocationTime", "LastFailedInvocationTime", "LastSuccessfulEvaluationTime", "LastFailedEvaluationTime", "FirstActivatedTime" ] sourcetype = self._config.get(tac.sourcetype, "aws:config:rule") last_times, events = [], [] for status in statuses: if status.get("LastSuccessfulEvaluationTime"): evt_time = tacommon.total_seconds( status["LastSuccessfulEvaluationTime"]) else: evt_time = "" ckpt_time = self._ckpt.last_evaluation_time( self._config[tac.region], self._config[tac.datainput], status["ConfigRuleName"]) last_times.append((evt_time, ckpt_time)) if ckpt_time == evt_time: continue for key in dkeys: if key in status: status[key] = "{}".format(status[key]) event = writer.create_event( index=self._config.get(tac.index, "default"), host=self._config.get(tac.host, ""), source=self._source_status, sourcetype=sourcetype, time=evt_time, unbroken=False, done=False, events=status, ) events.append(event) if events: writer.write_events(events) return last_times
def _index_run(self, data): etime = tacommon.total_seconds(data['completedAt']) event = self._writer.create_event( index=self._config.get(tac.index, 'default'), host=self._config.get(tac.host, ''), source=self._source, sourcetype=self._source_type, time=etime, unbroken=False, done=False, events=data, ) self._writer.write_events((event, ))
def _index_findings(self, findings): for item in findings: etime = tacommon.total_seconds(item['updatedAt']) event = self._writer.create_event( index=self._config.get(tac.index, "default"), host=self._config.get(tac.host, ""), source=self._source, sourcetype=self._source_type, time=etime, unbroken=False, done=False, events=item, ) self._writer.write_events((event, ))
def _index_compliance_details(self, rule_name): sourcetype = self._config.get(tac.sourcetype, "aws:config:rule") writer = self._config[tac.event_writer] next_token = "" while 1: response = self._client.get_compliance_details_by_config_rule( ConfigRuleName=rule_name, NextToken=next_token) if not tacommon.is_http_ok(response): logger.error( "Failed to collect compliance details for " "rule=%s, errorcode=%s", rule_name, tacommon.http_code(response)) return compliances = response.get("EvaluationResults") if not compliances: return events = [] for compliance in compliances: evt_time = compliance["ResultRecordedTime"] compliance["ResultRecordedTime"] = "{}".format(evt_time) compliance["ConfigRuleInvokedTime"] = "{}".format( compliance["ConfigRuleInvokedTime"]) compliance["EvaluationResultIdentifier"][ "OrderingTimestamp"] = "{}".format( compliance["EvaluationResultIdentifier"] ["OrderingTimestamp"]) evt_time = tacommon.total_seconds(evt_time) event = writer.create_event( index=self._config.get(tac.index, "default"), host=self._config.get(tac.host, ""), source=self._source_detail, sourcetype=sourcetype, time=evt_time, unbroken=False, done=False, events=compliance) events.append(event) writer.write_events(events) next_token = response.get("NextToken") if not next_token: return
def _index_data(self, records): writer = self._config[tac.event_writer] sourcetype = self._config.get(tac.sourcetype, "aws:cloudwatch") events = [] for dimension, datapoints in records: for datapoint in datapoints: datapoint.update(self._supplemental_data) datapoint[acc.metric_dimensions] = self._build_dimension_str( dimension) datapoint[acc.metric_name] = dimension["MetricName"] evt_time = tacommon.total_seconds(datapoint["Timestamp"]) del datapoint["Timestamp"] event = writer.create_event( index=self._config.get(tac.index, "default"), host=self._config.get(tac.host, ""), source=self._source, sourcetype=sourcetype, time=evt_time, unbroken=False, done=False, events=datapoint) events.append(event) writer.write_events(events, retry=10)
def _index_data(self, records): writer = self._config[tac.event_writer] sourcetype = self._config.get(tac.sourcetype, "aws:cloudwatch") events = [] use_metric_format = self._config[acc.use_metric_format] if use_metric_format: client = self._client.require_boto3_ec2_client() self._instances_data_list = self._get_instance_metadata(client) for dimension, datapoints in records: for datapoint in datapoints: datapoint.update(self._supplemental_data) evt_time = tacommon.total_seconds(datapoint["Timestamp"]) del datapoint["Timestamp"] if use_metric_format: metric_datapoint = self._create_metric_datapoint( dimension, datapoint) data = self._build_metric_str(metric_datapoint, evt_time) fields = metric_datapoint else: datapoint[ acc.metric_dimensions] = self._build_dimension_str( dimension) datapoint[acc.metric_name] = dimension["MetricName"] data = datapoint fields = None event = writer.create_event( index=self._config.get(tac.index, "default"), host=self._config.get(tac.host, ""), source=self._source, sourcetype=sourcetype, time=evt_time, unbroken=False, done=False, events=data, fields=fields) events.append(event) writer.write_events(events, retry=10)