def _index_compliance_summary(self, rule_name):
writer = self._config[tac.event_writer]
sourcetype = self._config.get(tac.sourcetype, "aws:config:rule")
response = self._client.get_compliance_summary_by_config_rule(
ConfigRuleName=rule_name)
if not tacommon.is_http_ok(response):
logger.error(
"Failed to collect compliance summary for "
"rule=%s, errorcode=%s", rule_name,
tacommon.http_code(response))
return
summary = response.get("ComplianceSummary")
if not summary:
return
evt_time = tacommon.total_seconds(
summary["ComplianceSummaryTimestamp"])
summary["ComplianceSummaryTimestamp"] = "{}".format(
summary["ComplianceSummaryTimestamp"])
summary["ConfigRuleName"] = rule_name
event = writer.create_event(index=self._config.get(
tac.index, "default"),
host=self._config.get(tac.host, ""),
source=self._source_summary,
sourcetype=sourcetype,
time=evt_time,
unbroken=False,
done=False,
events=summary)
writer.write_events((event, ))
def _last_evaluation_times(self, rule_name=""):
writer = self._config[tac.event_writer]
response = self._client.describe_config_rule_evaluation_status(
ConfigRuleNames=[rule_name])
if not tacommon.is_http_ok(response):
logger.error(
"Failed to describe config rule evaluation status, "
"errorcode=%s", tacommon.http_code(response))
return None
statuses = response.get("ConfigRulesEvaluationStatus")
if not statuses:
return None
dkeys = [
"LastSuccessfulInvocationTime", "LastFailedInvocationTime",
"LastSuccessfulEvaluationTime", "LastFailedEvaluationTime",
"FirstActivatedTime"
]
sourcetype = self._config.get(tac.sourcetype, "aws:config:rule")
last_times, events = [], []
for status in statuses:
if status.get("LastSuccessfulEvaluationTime"):
evt_time = tacommon.total_seconds(
status["LastSuccessfulEvaluationTime"])
else:
evt_time = ""
ckpt_time = self._ckpt.last_evaluation_time(
self._config[tac.region], self._config[tac.datainput],
status["ConfigRuleName"])
last_times.append((evt_time, ckpt_time))
if ckpt_time == evt_time:
continue
for key in dkeys:
if key in status:
status[key] = "{}".format(status[key])
event = writer.create_event(
index=self._config.get(tac.index, "default"),
host=self._config.get(tac.host, ""),
source=self._source_status,
sourcetype=sourcetype,
time=evt_time,
unbroken=False,
done=False,
events=status,
)
events.append(event)
if events:
writer.write_events(events)
return last_times
def _index_compliance_details(self, rule_name):
sourcetype = self._config.get(tac.sourcetype, "aws:config:rule")
writer = self._config[tac.event_writer]
next_token = ""
while 1:
response = self._client.get_compliance_details_by_config_rule(
ConfigRuleName=rule_name, NextToken=next_token)
if not tacommon.is_http_ok(response):
logger.error(
"Failed to collect compliance details for "
"rule=%s, errorcode=%s", rule_name,
tacommon.http_code(response))
return
compliances = response.get("EvaluationResults")
if not compliances:
return
events = []
for compliance in compliances:
evt_time = compliance["ResultRecordedTime"]
compliance["ResultRecordedTime"] = "{}".format(evt_time)
compliance["ConfigRuleInvokedTime"] = "{}".format(
compliance["ConfigRuleInvokedTime"])
compliance["EvaluationResultIdentifier"][
"OrderingTimestamp"] = "{}".format(
compliance["EvaluationResultIdentifier"]
["OrderingTimestamp"])
evt_time = tacommon.total_seconds(evt_time)
event = writer.create_event(
index=self._config.get(tac.index, "default"),
host=self._config.get(tac.host, ""),
source=self._source_detail,
sourcetype=sourcetype,
time=evt_time,
unbroken=False,
done=False,
events=compliance)
events.append(event)
writer.write_events(events)
next_token = response.get("NextToken")
if not next_token:
return
def _do_one_dimension(self, client, dimension, start_time, end_time):
if start_time == end_time:
return None
# perf_start = time.time()
logger.debug(
"Collect dimensions=%s, start_time=%s, end_time=%s for datainput=%s",
dimension, start_time, end_time, self._config[tac.datainput])
for i in xrange(4):
try:
response = client.get_metric_statistics(
Namespace=self._config[acc.metric_namespace],
MetricName=dimension["MetricName"],
Dimensions=dimension["Dimensions"],
StartTime=start_time,
EndTime=end_time,
Period=self._config[acc.period],
Statistics=self._config[acc.statistics])
except Exception as ex:
if "Rate exceeded" in ex.message:
tacommon.sleep_until(random.randint(20, 60), self.stopped)
logger.exception(
"Failed to get metrics for namespace=%s, dimension=%s,"
"metric_name=%s, datainput=%s, start_time=%s, "
"end_time=%s.", self._config[acc.metric_namespace],
dimension["Dimensions"], dimension["MetricName"],
self._config[tac.datainput], start_time, end_time)
self._handle_too_many_datapoints(ex, dimension)
tacommon.sleep_until(2**(i + 1), self.stopped)
else:
break
else:
return None
if not tacommon.is_http_ok(response):
logger.error(
"Failed to get metrics for namespace=%s, dimension=%s, "
"metric_name=%s, errorcode=%s.",
self._config[acc.metric_namespace], dimension["Dimensions"],
dimension["MetricName"], tacommon.http_code(response))
return None
# logger.debug("one_dimension_cost=%s", time.time() - perf_start)
return response.get("Datapoints")
def _list_rules(self, conf_info):
aws_account = self.callerArgs.data['aws_account'][0]
aws_iam_role = self.callerArgs.data.get('aws_iam_role', [None])[0]
region_name = self.callerArgs.data['aws_region'][0]
scheme, host, port = get_splunkd_access_info()
service = Service(scheme=scheme,
host=host,
port=port,
token=self.getSessionKey())
config = ConfigManager(service)
factory = AWSCredentialsProviderFactory(config)
provider = factory.create(aws_account, aws_iam_role)
credentials_cache = AWSCredentialsCache(provider)
client = credentials_cache.client('config', region_name)
all_rules = []
next_token = ""
while 1:
try:
response = client.describe_config_rules(NextToken=next_token)
except Exception as e:
logger.error('Failed to describe config rules')
msg = str(e.message)
logger.error(msg)
raise RestError(400, 'Failed to describe config rules: ' + msg)
if not tacommon.is_http_ok(response):
logger.error("Failed to describe config rules, errorcode=%s",
tacommon.http_code(response))
return
rules = response.get("ConfigRules")
if not rules:
break
all_rules.extend(rule["ConfigRuleName"] for rule in rules)
next_token = response.get("NextToken")
if not next_token:
break
for rule in all_rules:
conf_info[rule].append("rule_names", rule)
def _do_index_data(self):
if self._credentials.need_retire():
self._client, self._credentials = tacommon.get_service_client(
self._config, tac.config)
next_token = ""
rule_names = self._config.get("rule_names", [])
throttling = 0
while 1:
try:
response = self._client.describe_config_rules(
ConfigRuleNames=rule_names, NextToken=next_token)
except ClientError as e:
error_code = e.response["Error"].get("Code", "Unknown")
error_message = e.response["Error"].get("Message", "Unknown")
if error_code == "NoSuchConfigRuleException" and \
error_message.startswith('The ConfigRule'):
invalid_rule = error_message.split("'")[1]
rule_names.remove(invalid_rule)
logger.info("Neglect invalid rule and retry.",
invalid_rule=invalid_rule)
if rule_names:
next_token = ""
continue
else:
# empty rule_names will list all,
# directly returns to avoid
logger.info("No valid config rule found.",
region=self._config[tac.region],
datainput=self._config[tac.datainput])
return
elif error_code == "ThrottlingException":
logger.info(
"Throttling happened when DescribeConfigRules, "
"use exponential back off logic.")
time.sleep(2**(throttling + 1))
throttling += 1
else:
logger.exception(
"Unknown error code returned when describing config rules.",
region=self._config[tac.region],
datainput=self._config[tac.datainput])
return
except:
logger.exception(
"Unknown exception happened when describing config rules.",
region=self._config[tac.region],
datainput=self._config[tac.datainput])
return
if not tacommon.is_http_ok(response):
logger.error("Failed to describe config rules, errorcode=%s",
tacommon.http_code(response))
return
rules = response.get("ConfigRules")
if not rules:
return
self._index_rules(rules)
next_token = response.get("NextToken")
if not next_token:
return