def load(self):
workspace = self._app.workspace()
content = self._config.load('google_cloud_billing_inputs')
for name, fields in content.items():
parser = StanzaParser([
BooleanField('disabled',
default=False,
reverse=True,
rename='enabled'),
StringField('bucket_name', required=True),
StringField('report_prefix', required=True),
StringField('google_credentials_name',
required=True,
rename='profile'),
StringField('sourcetype', default='google:gcp:billing:report'),
DateTimeField('ingestion_start', default='1970-01-01'),
StringField('temp_file_folder', default=workspace),
IntegerField('polling_interval',
default=300,
rename='interval'),
StringField('index'),
])
params = parser.parse(fields)
if params.enabled:
yield name, params
def load(cls, config):
content = config.load('aws_settings', stanza='aws_sqs_based_s3')
parser = StanzaParser([
LogLevelField('log_level', default='WARNING')
])
settings = parser.parse(content)
return cls(settings)
def load(cls, config):
path = 'splunk_ta_google/google_settings'
content = config.load(path, stanza='global_settings', virtual=True)
parser = StanzaParser([LogLevelField('log_level')])
general = parser.parse(content)
content = config.load(path, stanza='proxy_settings', virtual=True)
parser = StanzaParser([
BooleanField("proxy_enabled", default=False, rename='enabled'),
StringField("proxy_type", rename='scheme', default='http'),
BooleanField("proxy_rdns", rename='rdns', default=False),
StringField("proxy_url", rename='host', default='127.0.0.1'),
StringField("proxy_port", rename='port', default='8080'),
StringField("proxy_username", rename='username', default=''),
StringField("proxy_password", rename='password', default=''),
])
proxy = parser.parse(content)
return cls(general, proxy)
def _load_aws_iam_role(self, aws_iam_role_name):
if not aws_iam_role_name:
raise AWSCredentialsError('The name of IAM role is invalid.')
name = 'splunk_ta_aws/settings/splunk_ta_aws_iam_role'
content = self._config.load(name, stanza=aws_iam_role_name, virtual=True)
parser = StanzaParser([StringField('arn')])
profile = parser.parse(content)
return AWSIAMRole(profile)
def create(cls, config, tenant_name):
content = config.load('splunk_ta_o365/tenants', tenant_name, virtual=True)
parser = StanzaParser([
StringField('endpoint'),
StringField('tenant_id'),
StringField('client_id'),
StringField('client_secret')
])
profile = parser.parse(content)
return O365Tenant(**vars(profile))
def load(cls, config):
content = config.load('splunk_ta_aws/splunk_ta_aws_settings_proxy', stanza='aws_proxy', virtual=True)
parser = StanzaParser([
BooleanField('proxy_enabled', rename='enabled'),
StringField('host'),
StringField('port'),
StringField('username'),
StringField('password')
])
settings = parser.parse(content)
logger.debug('Load proxy settings success.', **cls._wipe(settings))
return cls(settings)
def _load_aws_account(self, aws_account_name):
if not aws_account_name:
raise AWSCredentialsError('The name of account is invalid.')
name = 'splunk_ta_aws/settings/account'
content = self._config.load(name, stanza=aws_account_name, virtual=True)
parser = StanzaParser([
BooleanField('iam', default=False),
StringField('key_id'),
StringField('secret_key'),
IntegerField('category', default=0)
])
profile = parser.parse(content)
return AWSAccount(profile)
def perform(self, app, name, params):
Input.log(params, 'AWS SQS Input Collect')
parser = StanzaParser([
StringField('aws_account', required=True),
StringField('aws_iam_role'),
])
args = parser.parse(params)
aws_region = params['aws_region']
try:
config = app.create_config_service()
factory = AWSCredentialsProviderFactory(config)
provider = factory.create(args.aws_account, args.aws_iam_role)
credentials = AWSCredentialsCache(provider)
if credentials.need_retire(self._MIN_TTL):
credentials.refresh()
queue_url = get_sqs_queue_url(credentials, params['sqs_queue'],
aws_region)
Input.log(params, 'AWS SQS Input Collect TEST')
except Exception:
Input.log(params,
'Failed to get SQS queue url',
logging.ERROR,
error=traceback.format_exc())
return
sourcetype = params.get('sourcetype', 'aws:sqs')
index = params.get('index', 'default')
source = queue_url
portal = app.create_event_writer(index=index,
sourcetype=sourcetype,
source=source)
collector = SQSCollector(queue_url,
aws_region,
credentials,
logger,
handler=ingest,
portal=portal)
result = collector.run(app)
if result is not True:
Input.log(params, 'SQS queue fetching failed', logging.ERROR)
Input.log(params, 'SQS Fetching Finished')
def load(cls, config):
content = config.load('splunk_ta_o365_settings', stanza='logging')
parser = StanzaParser([LogLevelField('log_level', default='WARNING')])
settings = parser.parse(content)
return cls(settings)
def _load_assume_role_settings(self):
stanza = self._config.load('aws_settings', stanza='assume_role')
parser = StanzaParser([
IntegerField('duration', default=3600, lower=900, upper=3600)
])
return parser.parse(stanza)