def handleList(self, confInfo): # Set proxy for boto3 proxy = pc.get_proxy_info(self.getSessionKey()) tacommon.set_proxy_env(proxy) for queue_url in self._list_queues(): confInfo[queue_url].append('label', queue_url.split('/')[-1])
def get_tasks(self): conf_mgr = cm.ConfManager(self.metas[tac.server_uri], self.metas[tac.session_key]) tasks = self._get_config_rule_tasks(conf_mgr) settings = conf_mgr.all_stanzas_as_dicts(self.conf_file, do_reload=False) proxy_info = tpc.get_proxy_info(self.metas[tac.session_key]) # set proxy here for validating credentials tacommon.set_proxy_env(proxy_info) set_log_level(settings[tac.log_stanza][tac.log_level]) valid_tasks = [] for task in tasks: try: # validate credentials tacommon.get_service_client(task, tac.config) task[tac.log_level] = settings[tac.log_stanza][tac.log_level] task.update(settings[tac.global_settings]) task.update(proxy_info) valid_tasks.append(task) except Exception as e: input_name = scutil.extract_datainput_name(task[tac.name]) logger.exception( 'Failed to load credentials, ignore this input.', datainput=input_name) return tacommon.handle_hec(valid_tasks, "aws_config_rule")
def _get_region_host(self, session_key, payload): config = pc.get_proxy_info(session_key) tacommon.set_proxy_env(config) credentials_service = tacommon.create_credentials_service( get_splunkd_uri(), session_key) credentials = credentials_service.load( payload[tac.aws_account], payload[tac.aws_iam_role], ) config[tac.key_id] = credentials.aws_access_key_id config[tac.secret_key] = credentials.aws_secret_access_key config['aws_session_token'] = credentials.aws_session_token config[asc.bucket_name] = payload[asc.bucket_name] config[asc.host_name] = tac.CATEGORY_HOST_NAME_MAP[ credentials.category] if config[asc.host_name] == asc.default_host: region = get_region_for_bucketname(config) with open(ENDPOINTS_PATH, 'r') as endpoints_file: endpoints = json.load(endpoints_file) host_name = EndpointResolver(endpoints).construct_endpoint( 's3', region).get('hostname', asc.default_host) else: pattern = r's3[.-]([\w-]+)\.amazonaws.com' m = re.search(pattern, config[asc.host_name]) region = m.group(1) if m else 'us-east-1' host_name = config[asc.host_name] return (region, host_name)
def _get_tasks(self): if not self.stanza_configs: return None conf_mgr = cm.ConfManager(self.metas[tac.server_uri], self.metas[tac.session_key]) logging_settings = conf_mgr.get_stanza(self.log_info, asc.log_stanza, do_reload=False) # set the log level read from conf for our logger set_log_level(logging_settings[asc.log_level]) # entry point for this stanza task, setup root logger here # Generic S3 can be configured to be single-instance # or multiple instance # through env variable stanza_name = '' try: if len(self.stanza_configs) == 1: # only one stanza exists stanza_name = self.stanza_configs[0].get('name', '') except Exception: logger.exception('Failed to get stanza name!') stanza_name = extract_datainput_name(stanza_name) logging.setup_root_logger(app_name=tac.splunk_ta_aws, modular_name=asc.mod_name, stanza_name=stanza_name) proxy_info = tpc.get_proxy_info(self.metas[tac.session_key]) tasks, creds = [], {} for stanza in self.stanza_configs: task = {} task.update(stanza) task.update(self.metas) task.update(proxy_info) task[tac.log_level] = logging_settings[asc.log_level] task[tac.interval] = tacommon.get_interval(task, 3600) task[tac.polling_interval] = task[tac.interval] task[asc.max_retries] = int(task.get(asc.max_retries, 3)) task[asc.prefix] = task.get(asc.key_name) task[asc.last_modified] = self._get_last_modified_time( task[asc.initial_scan_datetime]) task[ asc. terminal_scan_datetime] = self._convert_terminal_scan_datetime( task.get(asc.terminal_scan_datetime)) input_name = scutil.extract_datainput_name(task[tac.name]) task[asc.data_input] = input_name task[tac.sourcetype] = task.get(tac.sourcetype, "aws:s3") task[asc.bucket_name] = str(task[asc.bucket_name]) if not task.get(asc.whitelist): task[asc.whitelist] = s3common.sourcetype_to_keyname_regex.get( task[tac.sourcetype]) tasks.append(task) logger.info("Done with configuration read from conf.") s3ckpt.handle_ckpts(tasks) return tasks
def handleList(self, confInfo): # Set proxy for boto3 proxy = pc.get_proxy_info(self.getSessionKey()) tacommon.set_proxy_env(proxy) queue_names = map( lambda queue_url: queue_url.split('/')[-1], self._list_queues(), ) for queue in queue_names: confInfo[queue].append('sqs_queue', queue)
def validate_keys(session_key, **account_info): # Set proxy proxy = pc.get_proxy_info(session_key) set_proxy_env(proxy) try: _get_caller_identity(account_info) except ClientError as e: if e.response['Error']['Code'] == 'InvalidClientTokenId': return False return True
def handleList(self, conf_info): logger.info("start listing config rules") for required in self.valid_params: if not self.callerArgs or not self.callerArgs.get(required): logger.error('Missing "%s"', required) raise Exception('Missing "{}"'.format(required)) # Set proxy for boto3 proxy = pc.get_proxy_info(self.getSessionKey()) tacommon.set_proxy_env(proxy) self._list_rules(conf_info) logger.info("end of listing config rules")
def _get_kinesis_tasks(self, conf_mgr): proxy_info = tpc.get_proxy_info(self.metas[tac.session_key]) stanzas = conf_mgr.all_stanzas(self.task_file, do_reload=False) tasks, creds = [], {} for stanza in stanzas: if scutil.is_true(stanza[tac.disabled]): continue # Normalize tac.account to tac.aws_account stanza[tac.aws_account] = stanza.get(tac.account) stanza[tac.aws_iam_role] = stanza.get(tac.aws_iam_role) tasks.extend(self._expand_tasks(stanza, creds, proxy_info)) return tasks
def create_s3_connection(key_id, secret_key, session_key, bucket_name=None, region=None, host_name=asc.default_host, aws_session_token=None): proxy_info = pc.get_proxy_info(session_key) proxy_info[tac.key_id] = key_id proxy_info[tac.secret_key] = secret_key proxy_info['aws_session_token'] = aws_session_token proxy_info[asc.host_name] = host_name proxy_info[asc.bucket_name] = bucket_name proxy_info[tac.region] = region return s3common.create_s3_connection(proxy_info)
def _prepare(self, splunkd_uri, session_key, aws_account, region, topic_name): # Set proxy proxy = pc.get_proxy_info(session_key) tacommon.set_proxy_env(proxy) if self._aws_account != aws_account: self._aws_account = aws_account self._client = get_aws_sns_client(splunkd_uri, session_key, aws_account, region) self._topic_name = topic_name self._topic_arn = get_aws_sns_topic_arn(self._client, topic_name) if self._aws_account != aws_account or self._topic_name != topic_name: self._topic_name = topic_name self._topic_arn = get_aws_sns_topic_arn(self._client, topic_name)
def get_tasks(self): conf_mgr = cm.ConfManager(self.metas[tac.server_uri], self.metas[tac.session_key]) tasks = self._get_cloudwatch_logs_tasks(conf_mgr) logging_settings = conf_mgr.get_stanza(self.conf_file, tac.log_stanza, do_reload=False) set_log_level(logging_settings[tac.log_level]) proxy_info = tpc.get_proxy_info(self.metas[tac.session_key]) for task in tasks: task[tac.log_level] = logging_settings[tac.log_level] task.update(proxy_info) return tasks
def handleList(self, confInfo): # Set proxy for boto3 proxy = pc.get_proxy_info(self.getSessionKey()) tacommon.set_proxy_env(proxy) try: attrs = query_queue_attributes( self.getSessionKey(), self.callerArgs.data['aws_account'][0], self.callerArgs.data.get('aws_iam_role', [None])[0], self.callerArgs.data['aws_region'][0], self.callerArgs.data['sqs_queue_url'][0], ) except ClientError as exc: RestHandlerError.ctl(400, msgx=exc) return confInfo['Attributes']['VisibilityTimeout'] = attrs.visibility_timeout confInfo['Attributes']['RedrivePolicy'] = attrs.redrive_policy
def handleList(self, confInfo): aws_account = None aws_account_category = tac.RegionCategory.COMMERCIAL if self.callerArgs['aws_account'] is not None: aws_account = self.callerArgs['aws_account'][0] aws_iam_role = None if self.callerArgs.get('aws_iam_role') is not None: aws_iam_role = self.callerArgs['aws_iam_role'][0] if not aws_account: confInfo['bucket_name'].append('bucket_name', []) return # Set proxy for boto3 proxy = pc.get_proxy_info(self.getSessionKey()) tacommon.set_proxy_env(proxy) cred_service = tacommon.create_credentials_service( get_splunkd_uri(), self.getSessionKey()) try: cred = cred_service.load(aws_account, aws_iam_role) aws_account_category = cred.category except ClientError as err: raise RestError(400, str(err.message) + '. Please make sure the AWS Account and Assume Role are correct.') host_name = tac.CATEGORY_HOST_NAME_MAP[aws_account_category] connection = connect_s3( cred.aws_access_key_id, cred.aws_secret_access_key, self.getSessionKey(), host_name, security_token=cred.aws_session_token, ) rs = timed(25, all_buckets, [], (connection,)) rlist = [] for r in rs: rlist.append(r.name) for bucket in rlist: confInfo[bucket].append('bucket_name', bucket) confInfo[bucket].append('host_name', host_name)
def load(splunkd_uri, session_key): logger.debug('AWS CloudTrail Input Configs') user, app = 'nobody', 'Splunk_TA_aws' configs = { key: load_config( make_splunk_endpoint(splunkd_uri, ep['endpoint'], user, app), session_key, ep['label']) for key, ep in Configs.ENDPOINTS.iteritems() } # handle IAM role for AWS accounts for _, item in configs[Configs.ACCOUNTS].iteritems(): if is_true(item.get('iam')): item['key_id'], item['secret_key'] = None, None configs[Configs.SETTINGS_PROXY] = \ pc.get_proxy_info(session_key) return configs
def get_account_id(account, session_key): # we can directly get account_id in EC2 Role if account.get('iam') and account.get('account_id'): return account.get('account_id') # Set proxy proxy = pc.get_proxy_info(session_key) set_proxy_env(proxy) # get arn arn = _get_caller_identity(account)['Arn'] match_results = re.findall(r"^arn:aws(-\S+)?:iam::(\d+):", arn) if len(match_results) == 1: partition_name, account_id = match_results[0] return account_id return None
def handleList(self, conf_info): logger.info("start listing kinesis streams") for required in self.valid_params: if not self.callerArgs or not self.callerArgs.get(required): logger.error('Missing "%s"', required) raise Exception('Missing "{}"'.format(required)) aws_account = "" if self.callerArgs[tac.account] is not None: aws_account = self.callerArgs[tac.account][0] aws_iam_role = None if self.callerArgs.get(tac.aws_iam_role) is not None: aws_iam_role = self.callerArgs[tac.aws_iam_role][0] # Set proxy for boto3 proxy = pc.get_proxy_info(self.getSessionKey()) tacommon.set_proxy_env(proxy) cred_service = tacommon.create_credentials_service( get_splunkd_uri(), self.getSessionKey()) cred = cred_service.load(aws_account, aws_iam_role) proxy[tac.server_uri] = get_splunkd_uri() proxy[tac.session_key] = self.getSessionKey() proxy[tac.aws_account] = aws_account proxy[tac.aws_iam_role] = aws_iam_role proxy[tac.region] = self.callerArgs[tac.region][0] proxy[tac.key_id] = cred.aws_access_key_id proxy[tac.secret_key] = cred.aws_secret_access_key proxy['aws_session_token'] = cred.aws_session_token client = akc.KinesisClient(proxy, logger) streams = client.list_streams() for stream in streams: conf_info[stream].append("stream_names", stream) logger.info("end of listing kinesis streams")
def get_tasks(self): conf_mgr = cm.ConfManager(self.metas[tac.server_uri], self.metas[tac.session_key]) tasks = self._get_description_tasks(conf_mgr) logging_settings = conf_mgr.get_stanza(self.conf_file, tac.log_stanza, do_reload=False) # set logging level for our logger set_log_level(logging_settings[tac.log_level]) proxy_info = tpc.get_proxy_info(self.metas[tac.session_key]) # Set proxy for loading credentials by boto3 tacommon.set_proxy_env(proxy_info) for task in tasks: task[tac.log_level] = logging_settings[tac.log_level] task.update(proxy_info) self._assign_source(tasks) return tasks
def get_tasks(self): conf_mgr = cm.ConfManager(self.metas[tac.server_uri], self.metas[tac.session_key]) stanzas = conf_mgr.all_stanzas(self.task_file, do_reload=False) settings = conf_mgr.all_stanzas_as_dicts(self.conf_file, do_reload=False) proxy_info = tpc.get_proxy_info(self.metas[tac.session_key]) # set proxy here for validating credentials tacommon.set_proxy_env(proxy_info) level = settings[tac.log_stanza][tac.log_level] set_log_level(level) tasks = self._get_inspector_tasks(stanzas, settings, proxy_info) config = dict() config.update(self.metas) config.update(settings[tac.global_settings]) _cleanup_checkpoints(tasks, config) tasks = [ task for task in tasks if not scutil.is_true(task.get('disabled')) ] return tacommon.handle_hec(tasks, "aws_inspector")
def get_tasks(self): if not self.stanza_configs: return None conf_mgr = cm.ConfManager(self.server_uri, self.session_key) settings = conf_mgr.all_stanzas_as_dicts(self.conf_file, do_reload=False) # set logging level for our logger set_log_level(settings[tac.log_stanza][tac.log_level]) proxy_info = tpc.get_proxy_info(self.session_key) tasks, creds = {}, {} for stanza in self.stanza_configs: input_name = scutil.extract_datainput_name(stanza[tac.name]) with logging.LogContext(datainput=input_name): stanza[tac.interval] = tacommon.get_interval(stanza, 60) stanza[tac.polling_interval] = stanza[tac.interval] stanza[acc.period] = int(stanza[acc.period]) if stanza[acc.period] > 86400 or stanza[acc.period] < 60: logger.error( "Granularity(period) is not in range[60, 86400], ignore this input.", Period=stanza[acc.period], ErrorCode="ConfigurationError", ErrorDetail= "Invalid Granularity(period). It's out of range [60, 86400]." ) continue if stanza[tac.polling_interval] % stanza[acc.period]: logger.error( "Polling interval is not multiple of period, ignore this input.", Period=stanza[acc.period], ErrorCode="ConfigurationError", ErrorDetail= "Polling interval should be a multiple of granularity(period)." ) continue stanza[tac.datainput] = input_name stanza[tac.sourcetype] = stanza.get(tac.sourcetype, "aws:cloudwatch") metric_names = stanza[acc.metric_names].strip() if metric_names != ".*": metric_names = json.loads(metric_names) else: metric_names = None stanza[acc.metric_names] = metric_names stanza[acc.metric_dimensions] = json.loads( stanza[acc.metric_dimensions]) stanza[acc.statistics] = json.loads(stanza[acc.statistics]) stanza[tac.log_level] = settings[tac.log_stanza][tac.log_level] stanza[tac.aws_account] = stanza.get('aws_account') stanza[tac.aws_iam_role] = stanza.get('aws_iam_role') stanza[acc.use_metric_format] = scutil.is_true( stanza.get(acc.use_metric_format, False)) stanza.update(self.metas) stanza.update(proxy_info) stanza.update(settings[tac.global_settings]) stanza[tac.use_hec] = scutil.is_true( stanza.get(tac.use_hec, False)) stanza[acc.max_api_saver_time] = \ int(stanza.get(acc.max_api_saver_time, 7200)) region_tasks = {} tasks[stanza[tac.datainput]] = region_tasks for region in stanza[tac.aws_region].split(","): region = region.strip() if not region: continue task = {} task.update(stanza) task[tac.aws_region] = region task[tac.region] = region num, rtasks = self._expand_task(task) if rtasks: region_tasks[region] = rtasks stanza[region] = num if not region_tasks: logger.warning("No metric/dimension has been found.") all_tasks = [] for region_tasks in tasks.itervalues(): for rtasks in region_tasks.itervalues(): all_tasks.extend(rtasks) tacommon.handle_hec(all_tasks, "aws_cloudwatch") return all_tasks