def handleList(self, confInfo):
# Set proxy for boto3
proxy = pc.get_proxy_info(self.getSessionKey())
tacommon.set_proxy_env(proxy)
for queue_url in self._list_queues():
confInfo[queue_url].append('label', queue_url.split('/')[-1])
def get_tasks(self):
conf_mgr = cm.ConfManager(self.metas[tac.server_uri],
self.metas[tac.session_key])
tasks = self._get_config_rule_tasks(conf_mgr)
settings = conf_mgr.all_stanzas_as_dicts(self.conf_file,
do_reload=False)
proxy_info = tpc.get_proxy_info(self.metas[tac.session_key])
# set proxy here for validating credentials
tacommon.set_proxy_env(proxy_info)
set_log_level(settings[tac.log_stanza][tac.log_level])
valid_tasks = []
for task in tasks:
try:
# validate credentials
tacommon.get_service_client(task, tac.config)
task[tac.log_level] = settings[tac.log_stanza][tac.log_level]
task.update(settings[tac.global_settings])
task.update(proxy_info)
valid_tasks.append(task)
except Exception as e:
input_name = scutil.extract_datainput_name(task[tac.name])
logger.exception(
'Failed to load credentials, ignore this input.',
datainput=input_name)
return tacommon.handle_hec(valid_tasks, "aws_config_rule")
def _get_region_host(self, session_key, payload):
config = pc.get_proxy_info(session_key)
tacommon.set_proxy_env(config)
credentials_service = tacommon.create_credentials_service(
get_splunkd_uri(), session_key)
credentials = credentials_service.load(
payload[tac.aws_account],
payload[tac.aws_iam_role],
)
config[tac.key_id] = credentials.aws_access_key_id
config[tac.secret_key] = credentials.aws_secret_access_key
config['aws_session_token'] = credentials.aws_session_token
config[asc.bucket_name] = payload[asc.bucket_name]
config[asc.host_name] = tac.CATEGORY_HOST_NAME_MAP[
credentials.category]
if config[asc.host_name] == asc.default_host:
region = get_region_for_bucketname(config)
with open(ENDPOINTS_PATH, 'r') as endpoints_file:
endpoints = json.load(endpoints_file)
host_name = EndpointResolver(endpoints).construct_endpoint(
's3', region).get('hostname', asc.default_host)
else:
pattern = r's3[.-]([\w-]+)\.amazonaws.com'
m = re.search(pattern, config[asc.host_name])
region = m.group(1) if m else 'us-east-1'
host_name = config[asc.host_name]
return (region, host_name)
def _get_tasks(self):
if not self.stanza_configs:
return None
conf_mgr = cm.ConfManager(self.metas[tac.server_uri],
self.metas[tac.session_key])
logging_settings = conf_mgr.get_stanza(self.log_info,
asc.log_stanza,
do_reload=False)
# set the log level read from conf for our logger
set_log_level(logging_settings[asc.log_level])
# entry point for this stanza task, setup root logger here
# Generic S3 can be configured to be single-instance
# or multiple instance
# through env variable
stanza_name = ''
try:
if len(self.stanza_configs) == 1:
# only one stanza exists
stanza_name = self.stanza_configs[0].get('name', '')
except Exception:
logger.exception('Failed to get stanza name!')
stanza_name = extract_datainput_name(stanza_name)
logging.setup_root_logger(app_name=tac.splunk_ta_aws,
modular_name=asc.mod_name,
stanza_name=stanza_name)
proxy_info = tpc.get_proxy_info(self.metas[tac.session_key])
tasks, creds = [], {}
for stanza in self.stanza_configs:
task = {}
task.update(stanza)
task.update(self.metas)
task.update(proxy_info)
task[tac.log_level] = logging_settings[asc.log_level]
task[tac.interval] = tacommon.get_interval(task, 3600)
task[tac.polling_interval] = task[tac.interval]
task[asc.max_retries] = int(task.get(asc.max_retries, 3))
task[asc.prefix] = task.get(asc.key_name)
task[asc.last_modified] = self._get_last_modified_time(
task[asc.initial_scan_datetime])
task[
asc.
terminal_scan_datetime] = self._convert_terminal_scan_datetime(
task.get(asc.terminal_scan_datetime))
input_name = scutil.extract_datainput_name(task[tac.name])
task[asc.data_input] = input_name
task[tac.sourcetype] = task.get(tac.sourcetype, "aws:s3")
task[asc.bucket_name] = str(task[asc.bucket_name])
if not task.get(asc.whitelist):
task[asc.whitelist] = s3common.sourcetype_to_keyname_regex.get(
task[tac.sourcetype])
tasks.append(task)
logger.info("Done with configuration read from conf.")
s3ckpt.handle_ckpts(tasks)
return tasks
def handleList(self, confInfo):
# Set proxy for boto3
proxy = pc.get_proxy_info(self.getSessionKey())
tacommon.set_proxy_env(proxy)
queue_names = map(
lambda queue_url: queue_url.split('/')[-1],
self._list_queues(),
)
for queue in queue_names:
confInfo[queue].append('sqs_queue', queue)
def validate_keys(session_key, **account_info):
# Set proxy
proxy = pc.get_proxy_info(session_key)
set_proxy_env(proxy)
try:
_get_caller_identity(account_info)
except ClientError as e:
if e.response['Error']['Code'] == 'InvalidClientTokenId':
return False
return True
def handleList(self, conf_info):
logger.info("start listing config rules")
for required in self.valid_params:
if not self.callerArgs or not self.callerArgs.get(required):
logger.error('Missing "%s"', required)
raise Exception('Missing "{}"'.format(required))
# Set proxy for boto3
proxy = pc.get_proxy_info(self.getSessionKey())
tacommon.set_proxy_env(proxy)
self._list_rules(conf_info)
logger.info("end of listing config rules")
def _get_kinesis_tasks(self, conf_mgr):
proxy_info = tpc.get_proxy_info(self.metas[tac.session_key])
stanzas = conf_mgr.all_stanzas(self.task_file, do_reload=False)
tasks, creds = [], {}
for stanza in stanzas:
if scutil.is_true(stanza[tac.disabled]):
continue
# Normalize tac.account to tac.aws_account
stanza[tac.aws_account] = stanza.get(tac.account)
stanza[tac.aws_iam_role] = stanza.get(tac.aws_iam_role)
tasks.extend(self._expand_tasks(stanza, creds, proxy_info))
return tasks
def create_s3_connection(key_id,
secret_key,
session_key,
bucket_name=None,
region=None,
host_name=asc.default_host,
aws_session_token=None):
proxy_info = pc.get_proxy_info(session_key)
proxy_info[tac.key_id] = key_id
proxy_info[tac.secret_key] = secret_key
proxy_info['aws_session_token'] = aws_session_token
proxy_info[asc.host_name] = host_name
proxy_info[asc.bucket_name] = bucket_name
proxy_info[tac.region] = region
return s3common.create_s3_connection(proxy_info)
def _prepare(self, splunkd_uri, session_key, aws_account, region,
topic_name):
# Set proxy
proxy = pc.get_proxy_info(session_key)
tacommon.set_proxy_env(proxy)
if self._aws_account != aws_account:
self._aws_account = aws_account
self._client = get_aws_sns_client(splunkd_uri, session_key,
aws_account, region)
self._topic_name = topic_name
self._topic_arn = get_aws_sns_topic_arn(self._client, topic_name)
if self._aws_account != aws_account or self._topic_name != topic_name:
self._topic_name = topic_name
self._topic_arn = get_aws_sns_topic_arn(self._client, topic_name)
def get_tasks(self):
conf_mgr = cm.ConfManager(self.metas[tac.server_uri],
self.metas[tac.session_key])
tasks = self._get_cloudwatch_logs_tasks(conf_mgr)
logging_settings = conf_mgr.get_stanza(self.conf_file,
tac.log_stanza,
do_reload=False)
set_log_level(logging_settings[tac.log_level])
proxy_info = tpc.get_proxy_info(self.metas[tac.session_key])
for task in tasks:
task[tac.log_level] = logging_settings[tac.log_level]
task.update(proxy_info)
return tasks
def handleList(self, confInfo):
# Set proxy for boto3
proxy = pc.get_proxy_info(self.getSessionKey())
tacommon.set_proxy_env(proxy)
try:
attrs = query_queue_attributes(
self.getSessionKey(),
self.callerArgs.data['aws_account'][0],
self.callerArgs.data.get('aws_iam_role', [None])[0],
self.callerArgs.data['aws_region'][0],
self.callerArgs.data['sqs_queue_url'][0],
)
except ClientError as exc:
RestHandlerError.ctl(400, msgx=exc)
return
confInfo['Attributes']['VisibilityTimeout'] = attrs.visibility_timeout
confInfo['Attributes']['RedrivePolicy'] = attrs.redrive_policy
def handleList(self, confInfo):
aws_account = None
aws_account_category = tac.RegionCategory.COMMERCIAL
if self.callerArgs['aws_account'] is not None:
aws_account = self.callerArgs['aws_account'][0]
aws_iam_role = None
if self.callerArgs.get('aws_iam_role') is not None:
aws_iam_role = self.callerArgs['aws_iam_role'][0]
if not aws_account:
confInfo['bucket_name'].append('bucket_name', [])
return
# Set proxy for boto3
proxy = pc.get_proxy_info(self.getSessionKey())
tacommon.set_proxy_env(proxy)
cred_service = tacommon.create_credentials_service(
get_splunkd_uri(), self.getSessionKey())
try:
cred = cred_service.load(aws_account, aws_iam_role)
aws_account_category = cred.category
except ClientError as err:
raise RestError(400, str(err.message) + '. Please make sure the AWS Account and Assume Role are correct.')
host_name = tac.CATEGORY_HOST_NAME_MAP[aws_account_category]
connection = connect_s3(
cred.aws_access_key_id, cred.aws_secret_access_key,
self.getSessionKey(), host_name,
security_token=cred.aws_session_token,
)
rs = timed(25, all_buckets, [], (connection,))
rlist = []
for r in rs:
rlist.append(r.name)
for bucket in rlist:
confInfo[bucket].append('bucket_name', bucket)
confInfo[bucket].append('host_name', host_name)
def load(splunkd_uri, session_key):
logger.debug('AWS CloudTrail Input Configs')
user, app = 'nobody', 'Splunk_TA_aws'
configs = {
key: load_config(
make_splunk_endpoint(splunkd_uri, ep['endpoint'], user, app),
session_key, ep['label'])
for key, ep in Configs.ENDPOINTS.iteritems()
}
# handle IAM role for AWS accounts
for _, item in configs[Configs.ACCOUNTS].iteritems():
if is_true(item.get('iam')):
item['key_id'], item['secret_key'] = None, None
configs[Configs.SETTINGS_PROXY] = \
pc.get_proxy_info(session_key)
return configs
def get_account_id(account, session_key):
# we can directly get account_id in EC2 Role
if account.get('iam') and account.get('account_id'):
return account.get('account_id')
# Set proxy
proxy = pc.get_proxy_info(session_key)
set_proxy_env(proxy)
# get arn
arn = _get_caller_identity(account)['Arn']
match_results = re.findall(r"^arn:aws(-\S+)?:iam::(\d+):", arn)
if len(match_results) == 1:
partition_name, account_id = match_results[0]
return account_id
return None
def handleList(self, conf_info):
logger.info("start listing kinesis streams")
for required in self.valid_params:
if not self.callerArgs or not self.callerArgs.get(required):
logger.error('Missing "%s"', required)
raise Exception('Missing "{}"'.format(required))
aws_account = ""
if self.callerArgs[tac.account] is not None:
aws_account = self.callerArgs[tac.account][0]
aws_iam_role = None
if self.callerArgs.get(tac.aws_iam_role) is not None:
aws_iam_role = self.callerArgs[tac.aws_iam_role][0]
# Set proxy for boto3
proxy = pc.get_proxy_info(self.getSessionKey())
tacommon.set_proxy_env(proxy)
cred_service = tacommon.create_credentials_service(
get_splunkd_uri(), self.getSessionKey())
cred = cred_service.load(aws_account, aws_iam_role)
proxy[tac.server_uri] = get_splunkd_uri()
proxy[tac.session_key] = self.getSessionKey()
proxy[tac.aws_account] = aws_account
proxy[tac.aws_iam_role] = aws_iam_role
proxy[tac.region] = self.callerArgs[tac.region][0]
proxy[tac.key_id] = cred.aws_access_key_id
proxy[tac.secret_key] = cred.aws_secret_access_key
proxy['aws_session_token'] = cred.aws_session_token
client = akc.KinesisClient(proxy, logger)
streams = client.list_streams()
for stream in streams:
conf_info[stream].append("stream_names", stream)
logger.info("end of listing kinesis streams")
def get_tasks(self):
conf_mgr = cm.ConfManager(self.metas[tac.server_uri],
self.metas[tac.session_key])
tasks = self._get_description_tasks(conf_mgr)
logging_settings = conf_mgr.get_stanza(self.conf_file,
tac.log_stanza,
do_reload=False)
# set logging level for our logger
set_log_level(logging_settings[tac.log_level])
proxy_info = tpc.get_proxy_info(self.metas[tac.session_key])
# Set proxy for loading credentials by boto3
tacommon.set_proxy_env(proxy_info)
for task in tasks:
task[tac.log_level] = logging_settings[tac.log_level]
task.update(proxy_info)
self._assign_source(tasks)
return tasks
def get_tasks(self):
conf_mgr = cm.ConfManager(self.metas[tac.server_uri],
self.metas[tac.session_key])
stanzas = conf_mgr.all_stanzas(self.task_file, do_reload=False)
settings = conf_mgr.all_stanzas_as_dicts(self.conf_file,
do_reload=False)
proxy_info = tpc.get_proxy_info(self.metas[tac.session_key])
# set proxy here for validating credentials
tacommon.set_proxy_env(proxy_info)
level = settings[tac.log_stanza][tac.log_level]
set_log_level(level)
tasks = self._get_inspector_tasks(stanzas, settings, proxy_info)
config = dict()
config.update(self.metas)
config.update(settings[tac.global_settings])
_cleanup_checkpoints(tasks, config)
tasks = [
task for task in tasks if not scutil.is_true(task.get('disabled'))
]
return tacommon.handle_hec(tasks, "aws_inspector")
def get_tasks(self):
if not self.stanza_configs:
return None
conf_mgr = cm.ConfManager(self.server_uri, self.session_key)
settings = conf_mgr.all_stanzas_as_dicts(self.conf_file,
do_reload=False)
# set logging level for our logger
set_log_level(settings[tac.log_stanza][tac.log_level])
proxy_info = tpc.get_proxy_info(self.session_key)
tasks, creds = {}, {}
for stanza in self.stanza_configs:
input_name = scutil.extract_datainput_name(stanza[tac.name])
with logging.LogContext(datainput=input_name):
stanza[tac.interval] = tacommon.get_interval(stanza, 60)
stanza[tac.polling_interval] = stanza[tac.interval]
stanza[acc.period] = int(stanza[acc.period])
if stanza[acc.period] > 86400 or stanza[acc.period] < 60:
logger.error(
"Granularity(period) is not in range[60, 86400], ignore this input.",
Period=stanza[acc.period],
ErrorCode="ConfigurationError",
ErrorDetail=
"Invalid Granularity(period). It's out of range [60, 86400]."
)
continue
if stanza[tac.polling_interval] % stanza[acc.period]:
logger.error(
"Polling interval is not multiple of period, ignore this input.",
Period=stanza[acc.period],
ErrorCode="ConfigurationError",
ErrorDetail=
"Polling interval should be a multiple of granularity(period)."
)
continue
stanza[tac.datainput] = input_name
stanza[tac.sourcetype] = stanza.get(tac.sourcetype,
"aws:cloudwatch")
metric_names = stanza[acc.metric_names].strip()
if metric_names != ".*":
metric_names = json.loads(metric_names)
else:
metric_names = None
stanza[acc.metric_names] = metric_names
stanza[acc.metric_dimensions] = json.loads(
stanza[acc.metric_dimensions])
stanza[acc.statistics] = json.loads(stanza[acc.statistics])
stanza[tac.log_level] = settings[tac.log_stanza][tac.log_level]
stanza[tac.aws_account] = stanza.get('aws_account')
stanza[tac.aws_iam_role] = stanza.get('aws_iam_role')
stanza[acc.use_metric_format] = scutil.is_true(
stanza.get(acc.use_metric_format, False))
stanza.update(self.metas)
stanza.update(proxy_info)
stanza.update(settings[tac.global_settings])
stanza[tac.use_hec] = scutil.is_true(
stanza.get(tac.use_hec, False))
stanza[acc.max_api_saver_time] = \
int(stanza.get(acc.max_api_saver_time, 7200))
region_tasks = {}
tasks[stanza[tac.datainput]] = region_tasks
for region in stanza[tac.aws_region].split(","):
region = region.strip()
if not region:
continue
task = {}
task.update(stanza)
task[tac.aws_region] = region
task[tac.region] = region
num, rtasks = self._expand_task(task)
if rtasks:
region_tasks[region] = rtasks
stanza[region] = num
if not region_tasks:
logger.warning("No metric/dimension has been found.")
all_tasks = []
for region_tasks in tasks.itervalues():
for rtasks in region_tasks.itervalues():
all_tasks.extend(rtasks)
tacommon.handle_hec(all_tasks, "aws_cloudwatch")
return all_tasks