def _create_bucket(self):
parser = StanzaParser([
StringField('bucket_name', required=True),
StringField('bucket_region', required=True),
])
args = self._extract_arguments(parser)
return S3Bucket(args.bucket_name, args.bucket_region)
def create_metadata(self):
stanza = self._kind + '://' + self._name
parser = StanzaParser([
StringField('index'),
StringField('host'),
StringField('stanza', fillempty=stanza)
])
return self._extract_arguments(parser)
def _create_options(self):
parser = StanzaParser([
StringField('report_prefix', default=u''),
StringField('temp_folder', default=None),
DateTimeField('start_date', default='1970-01', fmt='%Y-%m'),
StringField('report_names', default=r'.*')
])
return self._extract_arguments(parser)
def parse_extra(self):
parser = StanzaParser([
StringField('log_file_prefix', default=''),
DateTimeField('log_start_date', default='1970-1-1'),
StringField('log_name_format', default=''),
StringField('log_partitions', default=''),
])
return self._extract(parser)
def create_file_decoder(self):
parser = StanzaParser([
StringField('s3_file_decoder', rename='name', required=True),
StringField('sourcetype', default='')
])
args = self._extract_arguments(parser)
factory = DecoderFactory.create_default_instance()
return factory.create(**vars(args))
def _create_event_writer(self, app):
stanza = self._kind + '://' + self._name
parser = StanzaParser([
StringField('index'),
StringField('host'),
StringField('stanza', fillempty=stanza),
])
args = self._extract_arguments(parser)
return app.create_event_writer(**vars(args))
def _create_metadata(self):
stanza = self._kind + '://' + self._name
parser = StanzaParser([
StringField('index'),
StringField('host'),
StringField('stanza', fillempty=stanza),
StringField('sourcetype', default='o365:service:message')
])
return self._extract_arguments(parser)
def create_event_metadata(self, profile):
stanza_name = self._assemble_stanza_id()
parser = StanzaParser([
StringField('index'),
StringField('host'),
StringField('sourcetype', fillempty=profile.sourcetype),
StringField('stanza', fillempty=stanza_name)
])
return self._extract(parser)
def create_credentials(self, config):
parser = StanzaParser([
StringField('aws_account', required=True),
StringField('aws_iam_role'),
])
args = self._extract(parser)
factory = AWSCredentialsProviderFactory(config)
provider = factory.create(args.aws_account, args.aws_iam_role)
return AWSCredentialsCache(provider)
def create_sqs_agent(self, credential):
parser = StanzaParser([
StringField('sqs_queue_url', required=True),
StringField('sqs_queue_region', required=True),
IntegerField('sqs_batch_size', default=10, lower=1, upper=10),
])
args = self._extract_arguments(parser)
agent = SQSAgent(args.sqs_queue_url, args.sqs_queue_region, credential)
agent.set_batch_size(args.sqs_batch_size)
return agent
def _create_event_writer(self, app):
stanza = self._kind + '://' + self._name
parser = StanzaParser([
StringField('index'),
StringField('host'),
StringField('stanza', fillempty=stanza),
StringField('sourcetype', default='google:gcp:pubsub:message'),
])
args = self._extract_arguments(parser)
return app.create_event_writer(None, **vars(args))
def create(cls, config, tenant_name):
content = config.load('splunk_ta_o365/tenants', tenant_name, virtual=True)
parser = StanzaParser([
StringField('endpoint'),
StringField('tenant_id'),
StringField('client_id'),
StringField('client_secret')
])
profile = parser.parse(content)
return O365Tenant(**vars(profile))
def load(cls, config):
content = config.load('splunk_ta_aws/splunk_ta_aws_settings_proxy', stanza='aws_proxy', virtual=True)
parser = StanzaParser([
BooleanField('proxy_enabled', rename='enabled'),
StringField('host'),
StringField('port'),
StringField('username'),
StringField('password')
])
settings = parser.parse(content)
logger.debug('Load proxy settings success.', **cls._wipe(settings))
return cls(settings)
def _load_aws_account(self, aws_account_name):
if not aws_account_name:
raise AWSCredentialsError('The name of account is invalid.')
name = 'splunk_ta_aws/settings/account'
content = self._config.load(name, stanza=aws_account_name, virtual=True)
parser = StanzaParser([
BooleanField('iam', default=False),
StringField('key_id'),
StringField('secret_key'),
IntegerField('category', default=0)
])
profile = parser.parse(content)
return AWSAccount(profile)
def _create_subscription(self, mgmt):
parser = StanzaParser([
StringField('content_type'),
IntegerField('request_timeout', lower=10, upper=600, default=60)
])
args = self._extract_arguments(parser)
return mgmt.create_subscription(args.content_type, args.request_timeout)
def _create_credentials(self, config):
scopes = ['https://www.googleapis.com/auth/pubsub']
parser = StanzaParser(
[StringField('google_credentials_name', rename='profile')])
args = self._extract_arguments(parser)
factory = CredentialFactory(config)
return factory.load(args.profile, scopes)
def _create_subscriptions(self):
parser = StanzaParser([
StringField('google_project', required=True),
StringField('google_subscriptions', required=True),
])
args = self._extract_arguments(parser)
project = args.google_project
subscriptions = args.google_subscriptions
if ',' in subscriptions:
subscriptions = subscriptions.split(',')
subscriptions = [item.strip() for item in subscriptions]
else:
subscriptions = [subscriptions]
template = 'projects/{}/subscriptions/{}'
return [template.format(project, name) for name in subscriptions]
def load(cls, config):
path = 'splunk_ta_google/google_settings'
content = config.load(path, stanza='global_settings', virtual=True)
parser = StanzaParser([LogLevelField('log_level')])
general = parser.parse(content)
content = config.load(path, stanza='proxy_settings', virtual=True)
parser = StanzaParser([
BooleanField("proxy_enabled", default=False, rename='enabled'),
StringField("proxy_type", rename='scheme', default='http'),
BooleanField("proxy_rdns", rename='rdns', default=False),
StringField("proxy_url", rename='host', default='127.0.0.1'),
StringField("proxy_port", rename='port', default='8080'),
StringField("proxy_username", rename='username', default=''),
StringField("proxy_password", rename='password', default=''),
])
proxy = parser.parse(content)
return cls(general, proxy)
def _load_aws_iam_role(self, aws_iam_role_name):
if not aws_iam_role_name:
raise AWSCredentialsError('The name of IAM role is invalid.')
name = 'splunk_ta_aws/settings/splunk_ta_aws_iam_role'
content = self._config.load(name, stanza=aws_iam_role_name, virtual=True)
parser = StanzaParser([StringField('arn')])
profile = parser.parse(content)
return AWSIAMRole(profile)
def perform(self, app, name, params):
Input.log(params, 'AWS SQS Input Collect')
parser = StanzaParser([
StringField('aws_account', required=True),
StringField('aws_iam_role'),
])
args = parser.parse(params)
aws_region = params['aws_region']
try:
config = app.create_config_service()
factory = AWSCredentialsProviderFactory(config)
provider = factory.create(args.aws_account, args.aws_iam_role)
credentials = AWSCredentialsCache(provider)
if credentials.need_retire(self._MIN_TTL):
credentials.refresh()
queue_url = get_sqs_queue_url(credentials, params['sqs_queue'],
aws_region)
Input.log(params, 'AWS SQS Input Collect TEST')
except Exception:
Input.log(params,
'Failed to get SQS queue url',
logging.ERROR,
error=traceback.format_exc())
return
sourcetype = params.get('sourcetype', 'aws:sqs')
index = params.get('index', 'default')
source = queue_url
portal = app.create_event_writer(index=index,
sourcetype=sourcetype,
source=source)
collector = SQSCollector(queue_url,
aws_region,
credentials,
logger,
handler=ingest,
portal=portal)
result = collector.run(app)
if result is not True:
Input.log(params, 'SQS queue fetching failed', logging.ERROR)
Input.log(params, 'SQS Fetching Finished')
def _create_consumer(self, checkpoint, event_writer, portal, session):
parser = StanzaParser([
StringField('content_type'),
])
args = self._extract_arguments(parser)
content_type = args.content_type.lower()
if content_type.find('historical') != -1:
return ServiceHistoricalStatusConsumer(checkpoint, event_writer,
portal, session)
return ServiceCurrentStatusConsumer(event_writer, portal, session)
def load(self):
workspace = self._app.workspace()
content = self._config.load('google_cloud_billing_inputs')
for name, fields in content.items():
parser = StanzaParser([
BooleanField('disabled',
default=False,
reverse=True,
rename='enabled'),
StringField('bucket_name', required=True),
StringField('report_prefix', required=True),
StringField('google_credentials_name',
required=True,
rename='profile'),
StringField('sourcetype', default='google:gcp:billing:report'),
DateTimeField('ingestion_start', default='1970-01-01'),
StringField('temp_file_folder', default=workspace),
IntegerField('polling_interval',
default=300,
rename='interval'),
StringField('index'),
])
params = parser.parse(fields)
if params.enabled:
yield name, params
def _get_tenant_name(self):
parser = StanzaParser([
StringField('tenant_name'),
])
args = self._extract_arguments(parser)
return args.tenant_name
def create_event_writer(self, app):
metadata = self.create_metadata()
parser = StanzaParser([StringField('use_raw_hec')])
args = self._extract_arguments(parser)
url = args.use_raw_hec
return app.create_event_writer(url, **vars(metadata))
def create_region_cache(self, credentials):
parser = StanzaParser([
StringField('sqs_queue_region', required=True)
])
args = self._extract_arguments(parser)
return S3RegionCache(credentials, args.sqs_queue_region)
def _create_sourcetype(self):
parser = StanzaParser([
StringField('sourcetype', default='aws:billing:cur'),
])
args = self._extract_arguments(parser)
return args.sourcetype
