def _get_tasks(self):
if not self.stanza_configs:
return None
conf_mgr = cm.ConfManager(self.metas[tac.server_uri],
self.metas[tac.session_key])
logging_settings = conf_mgr.get_stanza(self.log_info,
asc.log_stanza,
do_reload=False)
# set the log level read from conf for our logger
set_log_level(logging_settings[asc.log_level])
# entry point for this stanza task, setup root logger here
# Generic S3 can be configured to be single-instance
# or multiple instance
# through env variable
stanza_name = ''
try:
if len(self.stanza_configs) == 1:
# only one stanza exists
stanza_name = self.stanza_configs[0].get('name', '')
except Exception:
logger.exception('Failed to get stanza name!')
stanza_name = extract_datainput_name(stanza_name)
logging.setup_root_logger(app_name=tac.splunk_ta_aws,
modular_name=asc.mod_name,
stanza_name=stanza_name)
proxy_info = tpc.get_proxy_info(self.metas[tac.session_key])
tasks, creds = [], {}
for stanza in self.stanza_configs:
task = {}
task.update(stanza)
task.update(self.metas)
task.update(proxy_info)
task[tac.log_level] = logging_settings[asc.log_level]
task[tac.interval] = tacommon.get_interval(task, 3600)
task[tac.polling_interval] = task[tac.interval]
task[asc.max_retries] = int(task.get(asc.max_retries, 3))
task[asc.prefix] = task.get(asc.key_name)
task[asc.last_modified] = self._get_last_modified_time(
task[asc.initial_scan_datetime])
task[
asc.
terminal_scan_datetime] = self._convert_terminal_scan_datetime(
task.get(asc.terminal_scan_datetime))
input_name = scutil.extract_datainput_name(task[tac.name])
task[asc.data_input] = input_name
task[tac.sourcetype] = task.get(tac.sourcetype, "aws:s3")
task[asc.bucket_name] = str(task[asc.bucket_name])
if not task.get(asc.whitelist):
task[asc.whitelist] = s3common.sourcetype_to_keyname_regex.get(
task[tac.sourcetype])
tasks.append(task)
logger.info("Done with configuration read from conf.")
s3ckpt.handle_ckpts(tasks)
return tasks
def get_tasks(self):
conf_mgr = cm.ConfManager(self.metas[tac.server_uri],
self.metas[tac.session_key])
tasks = self._get_config_rule_tasks(conf_mgr)
settings = conf_mgr.all_stanzas_as_dicts(self.conf_file,
do_reload=False)
proxy_info = tpc.get_proxy_info(self.metas[tac.session_key])
# set proxy here for validating credentials
tacommon.set_proxy_env(proxy_info)
set_log_level(settings[tac.log_stanza][tac.log_level])
valid_tasks = []
for task in tasks:
try:
# validate credentials
tacommon.get_service_client(task, tac.config)
task[tac.log_level] = settings[tac.log_stanza][tac.log_level]
task.update(settings[tac.global_settings])
task.update(proxy_info)
valid_tasks.append(task)
except Exception as e:
input_name = scutil.extract_datainput_name(task[tac.name])
logger.exception(
'Failed to load credentials, ignore this input.',
datainput=input_name)
return tacommon.handle_hec(valid_tasks, "aws_config_rule")
def convert_legacy_ckpt_to_new_ckpts(tasks):
conversion_done = "conversion_done"
conversion_ckpt_key = "aws_s3_ckpt_conversion_{}.ckpt"
store = create_state_store(tasks[0])
for stanza in tasks:
datainput = scutil.extract_datainput_name(stanza[tac.name])
ckpt_key = conversion_ckpt_key.format(datainput)
state = store.get_state(ckpt_key)
if state and state[conversion_done]:
logger.info("Ckpt conversion is already completed.",
datainput=datainput)
continue
legacy_key = _get_legacy_ckpt_key(stanza[tac.name],
stanza[asc.bucket_name])
legacy_state = store.get_state(legacy_key)
if not legacy_state:
logger.info("Legacy ckpt is not found.",
stanza=stanza[tac.name],
bucket_name=stanza[asc.bucket_name])
continue
convert_one_legacy_ckpt(store, stanza, legacy_state)
store.update_state(ckpt_key, {conversion_done: True})
def stream_events(self, inputs, ew):
"""overloaded splunklib modularinput method"""
# for multiple instance modinput, inputs dic got only one key
input_name = scutil.extract_datainput_name(inputs.inputs.keys()[0])
splunksdc.log.setup_root_logger(app_name="splunk_ta_aws",
modular_name='config',
stanza_name=input_name)
with splunksdc.log.LogContext(datainput=input_name):
self._stream_events(inputs, ew)
def __init__(self, splunkd_uri, session_key, ckpt_dir, input_name,
input_item, ew):
self.splunkd_uri, self.session_key = splunkd_uri, session_key
self.ckpt_dir = ckpt_dir
self.input_name, self.input_item = input_name, input_item
input_name = extract_datainput_name(self.input_name)
self.local_store = LocalKVService.create(
op.join(self.ckpt_dir, input_name + '.v3.ckpt'))
self.configs = Configs.load(self.splunkd_uri, self.session_key)
self.aws_account = None
self.sqs_client = None
ctcommon.event_writer = ew
def _expand_tasks(self, stanza):
tasks = []
regions = stanza[tac.regions].split(",")
for region in regions:
task = copy.copy(stanza)
task[tac.region] = region.strip()
task[tac.polling_interval] = int(stanza[tac.polling_interval])
task[tac.is_secure] = True
task[tac.datainput] = task[tac.stanza_name]
try:
tacommon.get_service_client(task, tac.inspector)
tasks.append(task)
except Exception as e:
input_name = scutil.extract_datainput_name(task[tac.name])
logger.exception(
'Failed to load credentials, ingore this input.',
datainput=input_name,
region=region)
return tasks
def get_tasks(self):
if not self.stanza_configs:
return None
conf_mgr = cm.ConfManager(self.metas[tac.server_uri],
self.metas[tac.session_key])
logging = conf_mgr.get_stanza(self.log_info,
asc.log_stanza,
do_reload=True)
proxy_info = tpc.get_proxy_info(self.metas[tac.session_key])
tasks, creds = [], {}
for stanza in self.stanza_configs:
task = {}
task.update(stanza)
task.update(self.metas)
task.update(proxy_info)
key_id, secret_key = tacommon.get_aws_creds(
stanza, self.metas, creds)
task[tac.log_level] = logging[asc.log_level]
task[tac.key_id] = key_id
task[tac.secret_key] = secret_key
task[tac.interval] = tacommon.get_interval(task, 3600)
task[asc.max_retries] = int(task.get(asc.max_retries, 10))
task[asc.prefix] = task.get(asc.key_name)
task[asc.last_modified] = self._get_last_modified_time(
task[asc.initial_scan_datetime])
input_name = scutil.extract_datainput_name(task[tac.name])
task[asc.data_input] = input_name
task[tac.sourcetype] = task.get(tac.sourcetype, "aws:s3")
task[asc.bucket_name] = str(task[asc.bucket_name])
if not task.get(asc.whitelist):
task[asc.whitelist] = s3common.sourcetype_to_keyname_regex.get(
task[tac.sourcetype])
tasks.append(task)
s3ckpt.handle_ckpts(tasks)
return tasks
def get_tasks(self):
if not self.stanza_configs:
return None
conf_mgr = cm.ConfManager(self.server_uri, self.session_key)
settings = conf_mgr.all_stanzas_as_dicts(self.conf_file,
do_reload=False)
# set logging level for our logger
set_log_level(settings[tac.log_stanza][tac.log_level])
proxy_info = tpc.get_proxy_info(self.session_key)
tasks, creds = {}, {}
for stanza in self.stanza_configs:
input_name = scutil.extract_datainput_name(stanza[tac.name])
with logging.LogContext(datainput=input_name):
stanza[tac.interval] = tacommon.get_interval(stanza, 60)
stanza[tac.polling_interval] = stanza[tac.interval]
stanza[acc.period] = int(stanza[acc.period])
if stanza[acc.period] > 86400 or stanza[acc.period] < 60:
logger.error(
"Granularity(period) is not in range[60, 86400], ignore this input.",
Period=stanza[acc.period],
ErrorCode="ConfigurationError",
ErrorDetail=
"Invalid Granularity(period). It's out of range [60, 86400]."
)
continue
if stanza[tac.polling_interval] % stanza[acc.period]:
logger.error(
"Polling interval is not multiple of period, ignore this input.",
Period=stanza[acc.period],
ErrorCode="ConfigurationError",
ErrorDetail=
"Polling interval should be a multiple of granularity(period)."
)
continue
stanza[tac.datainput] = input_name
stanza[tac.sourcetype] = stanza.get(tac.sourcetype,
"aws:cloudwatch")
metric_names = stanza[acc.metric_names].strip()
if metric_names != ".*":
metric_names = json.loads(metric_names)
else:
metric_names = None
stanza[acc.metric_names] = metric_names
stanza[acc.metric_dimensions] = json.loads(
stanza[acc.metric_dimensions])
stanza[acc.statistics] = json.loads(stanza[acc.statistics])
stanza[tac.log_level] = settings[tac.log_stanza][tac.log_level]
stanza[tac.aws_account] = stanza.get('aws_account')
stanza[tac.aws_iam_role] = stanza.get('aws_iam_role')
stanza[acc.use_metric_format] = scutil.is_true(
stanza.get(acc.use_metric_format, False))
stanza.update(self.metas)
stanza.update(proxy_info)
stanza.update(settings[tac.global_settings])
stanza[tac.use_hec] = scutil.is_true(
stanza.get(tac.use_hec, False))
stanza[acc.max_api_saver_time] = \
int(stanza.get(acc.max_api_saver_time, 7200))
region_tasks = {}
tasks[stanza[tac.datainput]] = region_tasks
for region in stanza[tac.aws_region].split(","):
region = region.strip()
if not region:
continue
task = {}
task.update(stanza)
task[tac.aws_region] = region
task[tac.region] = region
num, rtasks = self._expand_task(task)
if rtasks:
region_tasks[region] = rtasks
stanza[region] = num
if not region_tasks:
logger.warning("No metric/dimension has been found.")
all_tasks = []
for region_tasks in tasks.itervalues():
for rtasks in region_tasks.itervalues():
all_tasks.extend(rtasks)
tacommon.handle_hec(all_tasks, "aws_cloudwatch")
return all_tasks
def get_key_ckpt_key(data_input, bucket_name, key_name):
data_input = scutil.extract_datainput_name(data_input)
encoded = hashlib.md5("{data_input}${bucket_name}${key_name}".format(
data_input=data_input, bucket_name=bucket_name, key_name=key_name))
return encoded.hexdigest()
