def _create_credentials(self, config): scopes = ['https://www.googleapis.com/auth/pubsub'] parser = StanzaParser( [StringField('google_credentials_name', rename='profile')]) args = self._extract_arguments(parser) factory = CredentialFactory(config) return factory.load(args.profile, scopes)
def _create_bucket(self): parser = StanzaParser([ StringField('bucket_name', required=True), StringField('bucket_region', required=True), ]) args = self._extract_arguments(parser) return S3Bucket(args.bucket_name, args.bucket_region)
def load(cls, config): content = config.load('aws_settings', stanza='aws_sqs_based_s3') parser = StanzaParser([ LogLevelField('log_level', default='WARNING') ]) settings = parser.parse(content) return cls(settings)
def load(self): workspace = self._app.workspace() content = self._config.load('google_cloud_billing_inputs') for name, fields in content.items(): parser = StanzaParser([ BooleanField('disabled', default=False, reverse=True, rename='enabled'), StringField('bucket_name', required=True), StringField('report_prefix', required=True), StringField('google_credentials_name', required=True, rename='profile'), StringField('sourcetype', default='google:gcp:billing:report'), DateTimeField('ingestion_start', default='1970-01-01'), StringField('temp_file_folder', default=workspace), IntegerField('polling_interval', default=300, rename='interval'), StringField('index'), ]) params = parser.parse(fields) if params.enabled: yield name, params
def create_batch_executor(self): parser = StanzaParser([ IntegerField('sqs_batch_size', rename='number_of_threads', default=10, lower=1, upper=10), ]) args = self._extract_arguments(parser) return BatchExecutor(number_of_threads=args.number_of_threads)
def _create_subscription(self, mgmt): parser = StanzaParser([ StringField('content_type'), IntegerField('request_timeout', lower=10, upper=600, default=60) ]) args = self._extract_arguments(parser) return mgmt.create_subscription(args.content_type, args.request_timeout)
def parse_extra(self): parser = StanzaParser([ StringField('log_file_prefix', default=''), DateTimeField('log_start_date', default='1970-1-1'), StringField('log_name_format', default=''), StringField('log_partitions', default=''), ]) return self._extract(parser)
def parse_options(self): parser = StanzaParser([ IntegerField('max_retries', default=-1, lower=-1, upper=1000), IntegerField('max_fails', default=10000, lower=0, upper=10000), IntegerField('max_number_of_process', default=2, lower=1, upper=64), IntegerField('max_number_of_thread', default=4, lower=1, upper=64) ]) return self._extract(parser)
def load(cls, config): path = 'splunk_ta_google/google_settings' content = config.load(path, stanza='global_settings', virtual=True) parser = StanzaParser([LogLevelField('log_level')]) general = parser.parse(content) content = config.load(path, stanza='proxy_settings', virtual=True) parser = StanzaParser([ BooleanField("proxy_enabled", default=False, rename='enabled'), StringField("proxy_type", rename='scheme', default='http'), BooleanField("proxy_rdns", rename='rdns', default=False), StringField("proxy_url", rename='host', default='127.0.0.1'), StringField("proxy_port", rename='port', default='8080'), StringField("proxy_username", rename='username', default=''), StringField("proxy_password", rename='password', default=''), ]) proxy = parser.parse(content) return cls(general, proxy)
def create_metadata(self): stanza = self._kind + '://' + self._name parser = StanzaParser([ StringField('index'), StringField('host'), StringField('stanza', fillempty=stanza) ]) return self._extract_arguments(parser)
def _create_options(self): parser = StanzaParser([ StringField('report_prefix', default=u''), StringField('temp_folder', default=None), DateTimeField('start_date', default='1970-01', fmt='%Y-%m'), StringField('report_names', default=r'.*') ]) return self._extract_arguments(parser)
def create_file_decoder(self): parser = StanzaParser([ StringField('s3_file_decoder', rename='name', required=True), StringField('sourcetype', default='') ]) args = self._extract_arguments(parser) factory = DecoderFactory.create_default_instance() return factory.create(**vars(args))
def _create_metadata(self): stanza = self._kind + '://' + self._name parser = StanzaParser([ StringField('index'), StringField('host'), StringField('stanza', fillempty=stanza), StringField('sourcetype', default='o365:service:message') ]) return self._extract_arguments(parser)
def _load_aws_iam_role(self, aws_iam_role_name): if not aws_iam_role_name: raise AWSCredentialsError('The name of IAM role is invalid.') name = 'splunk_ta_aws/settings/splunk_ta_aws_iam_role' content = self._config.load(name, stanza=aws_iam_role_name, virtual=True) parser = StanzaParser([StringField('arn')]) profile = parser.parse(content) return AWSIAMRole(profile)
def create_credentials(self, config): parser = StanzaParser([ StringField('aws_account', required=True), StringField('aws_iam_role'), ]) args = self._extract(parser) factory = AWSCredentialsProviderFactory(config) provider = factory.create(args.aws_account, args.aws_iam_role) return AWSCredentialsCache(provider)
def create_event_metadata(self, profile): stanza_name = self._assemble_stanza_id() parser = StanzaParser([ StringField('index'), StringField('host'), StringField('sourcetype', fillempty=profile.sourcetype), StringField('stanza', fillempty=stanza_name) ]) return self._extract(parser)
def _create_event_writer(self, app): stanza = self._kind + '://' + self._name parser = StanzaParser([ StringField('index'), StringField('host'), StringField('stanza', fillempty=stanza), ]) args = self._extract_arguments(parser) return app.create_event_writer(**vars(args))
def create_sqs_agent(self, credential): parser = StanzaParser([ StringField('sqs_queue_url', required=True), StringField('sqs_queue_region', required=True), IntegerField('sqs_batch_size', default=10, lower=1, upper=10), ]) args = self._extract_arguments(parser) agent = SQSAgent(args.sqs_queue_url, args.sqs_queue_region, credential) agent.set_batch_size(args.sqs_batch_size) return agent
def _create_consumer(self, checkpoint, event_writer, portal, session): parser = StanzaParser([ StringField('content_type'), ]) args = self._extract_arguments(parser) content_type = args.content_type.lower() if content_type.find('historical') != -1: return ServiceHistoricalStatusConsumer(checkpoint, event_writer, portal, session) return ServiceCurrentStatusConsumer(event_writer, portal, session)
def create(cls, config, tenant_name): content = config.load('splunk_ta_o365/tenants', tenant_name, virtual=True) parser = StanzaParser([ StringField('endpoint'), StringField('tenant_id'), StringField('client_id'), StringField('client_secret') ]) profile = parser.parse(content) return O365Tenant(**vars(profile))
def _create_event_writer(self, app): stanza = self._kind + '://' + self._name parser = StanzaParser([ StringField('index'), StringField('host'), StringField('stanza', fillempty=stanza), StringField('sourcetype', default='google:gcp:pubsub:message'), ]) args = self._extract_arguments(parser) return app.create_event_writer(None, **vars(args))
def create_s3_agent(self, credential): _1MB = 1024 * 1024 _8MB = _1MB * 8 _64MB = _8MB * 8 parser = StanzaParser([ IntegerField('s3_multipart_threshold', default=_8MB, lower=_8MB, upper=_64MB) ]) args = self._extract_arguments(parser) agent = S3Agent(credential) agent.set_multipart_threshold(args.s3_multipart_threshold) return agent
def load(cls, config): content = config.load('splunk_ta_aws/splunk_ta_aws_settings_proxy', stanza='aws_proxy', virtual=True) parser = StanzaParser([ BooleanField('proxy_enabled', rename='enabled'), StringField('host'), StringField('port'), StringField('username'), StringField('password') ]) settings = parser.parse(content) logger.debug('Load proxy settings success.', **cls._wipe(settings)) return cls(settings)
def _load_aws_account(self, aws_account_name): if not aws_account_name: raise AWSCredentialsError('The name of account is invalid.') name = 'splunk_ta_aws/settings/account' content = self._config.load(name, stanza=aws_account_name, virtual=True) parser = StanzaParser([ BooleanField('iam', default=False), StringField('key_id'), StringField('secret_key'), IntegerField('category', default=0) ]) profile = parser.parse(content) return AWSAccount(profile)
def _create_subscriptions(self): parser = StanzaParser([ StringField('google_project', required=True), StringField('google_subscriptions', required=True), ]) args = self._extract_arguments(parser) project = args.google_project subscriptions = args.google_subscriptions if ',' in subscriptions: subscriptions = subscriptions.split(',') subscriptions = [item.strip() for item in subscriptions] else: subscriptions = [subscriptions] template = 'projects/{}/subscriptions/{}' return [template.format(project, name) for name in subscriptions]
def perform(self, app, name, params): Input.log(params, 'AWS SQS Input Collect') parser = StanzaParser([ StringField('aws_account', required=True), StringField('aws_iam_role'), ]) args = parser.parse(params) aws_region = params['aws_region'] try: config = app.create_config_service() factory = AWSCredentialsProviderFactory(config) provider = factory.create(args.aws_account, args.aws_iam_role) credentials = AWSCredentialsCache(provider) if credentials.need_retire(self._MIN_TTL): credentials.refresh() queue_url = get_sqs_queue_url(credentials, params['sqs_queue'], aws_region) Input.log(params, 'AWS SQS Input Collect TEST') except Exception: Input.log(params, 'Failed to get SQS queue url', logging.ERROR, error=traceback.format_exc()) return sourcetype = params.get('sourcetype', 'aws:sqs') index = params.get('index', 'default') source = queue_url portal = app.create_event_writer(index=index, sourcetype=sourcetype, source=source) collector = SQSCollector(queue_url, aws_region, credentials, logger, handler=ingest, portal=portal) result = collector.run(app) if result is not True: Input.log(params, 'SQS queue fetching failed', logging.ERROR) Input.log(params, 'SQS Fetching Finished')
def _get_tenant_name(self): parser = StanzaParser([ StringField('tenant_name'), ]) args = self._extract_arguments(parser) return args.tenant_name
def parse_options(self): parser = StanzaParser([ IntegerField('max_receive_count', default=-1), IntegerField('exit_on_idle', default=15), ]) return self._extract_arguments(parser)
def create_region_cache(self, credentials): parser = StanzaParser([ StringField('sqs_queue_region', required=True) ]) args = self._extract_arguments(parser) return S3RegionCache(credentials, args.sqs_queue_region)
def create_event_writer(self, app): metadata = self.create_metadata() parser = StanzaParser([StringField('use_raw_hec')]) args = self._extract_arguments(parser) url = args.use_raw_hec return app.create_event_writer(url, **vars(metadata))