def to_python(self, value, session_key=None):
Field.to_python(self, value, session_key=None)
# Parse the size
m = DataSizeField.DATA_SIZE_RE.match(value)
# Make sure the duration could be parsed
if m is None:
raise FieldValidationException("The value of '%s' for the '%s' parameter is not a valid size of data" % (str(value), self.name))
# Get the units and duration
d = m.groupdict()
units = d['units'].lower()
# Parse the value provided
try:
size = int(d['size'])
except ValueError:
raise FieldValidationException("The size '%s' for the '%s' parameter is not a valid number" % (d['duration'], self.name))
# Make sure the units are valid
if len(units) > 0 and units not in DataSizeField.UNITS:
raise FieldValidationException("The unit '%s' for the '%s' parameter is not a valid unit of duration" % (units, self.name))
# Convert the units to seconds
if len(units) > 0:
return size * DataSizeField.UNITS[units]
else:
return size
def to_python(self, value):
Field.to_python(self, value)
# Parse the duration
m = DurationField.DURATION_RE.match(value)
# Make sure the duration could be parsed
if m is None:
raise FieldValidationException("The value of '%s' for the '%s' parameter is not a valid duration" % (str(value), self.name))
# Get the units and duration
d = m.groupdict()
units = d['units']
# Parse the value provided
try:
duration = int(d['duration'])
except ValueError:
raise FieldValidationException("The duration '%s' for the '%s' parameter is not a valid number" % (d['duration'], self.name))
# Make sure the units are valid
if len(units) > 0 and units not in DurationField.UNITS:
raise FieldValidationException("The unit '%s' for the '%s' parameter is not a valid unit of duration" % (units, self.name))
# Convert the units to seconds
if len(units) > 0:
return duration * DurationField.UNITS[units]
else:
return duration
def __init__(self):
scheme_args = {
'title': "Google Spreadsheet",
'description':
"Allows you to import/export Splunk lookups to/from Google spreadsheets",
'use_external_validation': "true",
'streaming_mode': "xml",
'use_single_instance': "true"
}
args = [
Field("spreadsheet",
"Spreadsheet Title",
"The title of the spreadsheet",
empty_allowed=False),
Field("worksheet",
"Worksheet Name",
'The name of the worksheet (e.g. "Sheet1")',
empty_allowed=False),
Field(
"service_account_key_file",
"OAuth2 Service Account Key File",
'The service account key with the credentials necessary for authenticating to Google',
empty_allowed=False,
required_on_create=False,
required_on_edit=False),
BooleanField(
"only_if_changed",
"Import file only if changed",
"If set to true, then the import will only be done if the Google worksheet was changed.",
empty_allowed=True,
required_on_create=False,
required_on_edit=False),
Field(
"operation",
"Operation",
"The operation to perform (import into Splunk or export to Google Drive)",
empty_allowed=False),
Field("lookup_name",
"Lookup File Name",
'The name of the lookup file to import the content into',
empty_allowed=False),
DurationField(
"interval",
"Interval",
"The interval defining how often to import the file; can include time units (e.g. 15m for 15 minutes, 8h for 8 hours)",
empty_allowed=False),
DeprecatedField("google_login", "Google Login",
'The login to use when authenticating to Google'),
DeprecatedField(
"google_password", "Google Password",
'The password to use when authenticating to Google. You will need to use an app-specific password here if you are using two-factor authentication.'
)
]
ModularInput.__init__(self,
scheme_args,
args,
logger_name='google_spreadsheet_modular_input')
def __init__(self, timeout=30, **kwargs):
scheme_args = {
'title': "JWT Webhook",
'description': "Retrieve data from jwt webhook using SSL",
'use_single_instance': True
}
args = [
IntegerField('port',
'Port',
'The port to run the web-server on',
none_allowed=False,
empty_allowed=False),
Field(
'secret',
'Secret',
'The secret key to decode the JWT encoded payload, leave it empty if the payload is not JWT encoded.',
none_allowed=True,
empty_allowed=True),
Field(
'path',
'Path',
'A wildcard that the path of requests must match (paths generally begin with a "/" and can include a wildcard)',
none_allowed=True,
empty_allowed=True),
FilePathField(
'key_file',
'SSL Certificate Key File',
'The path to the SSL certificate key file (if the certificate requires a key); typically uses .KEY file extension',
none_allowed=True,
empty_allowed=True,
validate_file_existence=True),
FilePathField(
'cert_file',
'SSL Certificate File',
'The path to the SSL certificate file (if you want to use encryption); typically uses .DER, .PEM, .CRT, .CER file extensions',
none_allowed=False,
empty_allowed=False,
validate_file_existence=True),
Field(
'password',
'Password',
'The password to decrypt the private key, leave it empty if the private key is not encrypted.',
none_allowed=True,
empty_allowed=True),
]
ModularInput.__init__(self,
scheme_args,
args,
logger_name="webhook_modular_input",
sleep_interval=60)
if timeout > 0:
self.timeout = timeout
else:
self.timeout = 30
self.http_daemons = {}
def to_python(self, value, session_key=None):
Field.to_python(self, value, session_key)
if value is None:
return None
try:
ports_list = parseIntSet(value, True)
except ValueError:
raise FieldValidationException("The value of '%s' for the '%s' parameter is not a valid list" % (str(value), self.name))
return ports_list
def __init__(self, timeout=None, thread_limit=None):
scheme_args = {'title': "Website Availability Check",
'description': "Connects to a website in order to obtain performance statistics",
'use_external_validation': "true",
'streaming_mode': "xml",
'use_single_instance': "true"}
args = [
Field("title", "Title", "A short description (typically just the domain name)", empty_allowed=False),
URLField("url", "URL", "The URL to connect to (must be be either HTTP or HTTPS protocol)", empty_allowed=False, require_https_on_cloud=True),
DurationField("interval", "Interval", "The interval defining how often to perform the check; can include time units (e.g. 15m for 15 minutes, 8h for 8 hours)", empty_allowed=False),
Field("configuration", "Configuration", "Defines a specific proxy configuration to use (in website_monitoring.spec) if not using the default; only used if you want to have multiple proxy servers", none_allowed=True, empty_allowed=True),
Field("client_certificate", "Client Certificate Path", "Defines the path to the client certificate (if the website requires client SSL authentication)", none_allowed=True, empty_allowed=True),
Field("client_certificate_key", "Client Certificate Key Path", "Defines the path to the client certificate key (necessary of the key is in a separate file from the certificate)", none_allowed=True, empty_allowed=True),
Field("username", "Username", "The username to use for authenticating (only HTTP authentication supported)", none_allowed=True, empty_allowed=True, required_on_create=False, required_on_edit=False),
Field("password", "Password", "The password to use for authenticating (only HTTP authentication supported)", none_allowed=True, empty_allowed=True, required_on_create=False, required_on_edit=False),
Field("user_agent", "User Agent", "The user-agent to use when communicating with the server", none_allowed=True, empty_allowed=True, required_on_create=False, required_on_edit=False),
Field("should_contain_string", "String match", "A string that should be present in the content", none_allowed=True, empty_allowed=True, required_on_create=False, required_on_edit=False)
]
ModularInput.__init__(self, scheme_args, args, logger_name='web_availability_modular_input', logger_level=logging.DEBUG)
if timeout > 0:
self.timeout = timeout
else:
self.timeout = 30
if thread_limit is None:
self.thread_limit = WebPing.DEFAULT_THREAD_LIMIT
else:
self.thread_limit = thread_limit
self.threads = {}
def __init__(self, timeout=30):
scheme_args = {
'title':
"PCAP",
'description':
"Watch directories for packet capture files (*.pcap) and process them using Bro."
}
args = [
Field("path",
"Path",
"Specify where the pcap files are stored (eg: /var/pcap).",
empty_allowed=False),
BooleanField(
"recursive",
"Recursive",
"Specify if splunk should monitor all sub directories for incoming pcap. True or False.",
empty_allowed=False),
Field(
"store_dir",
"Log directory",
"Specify where the created log files by Bro will be stored (eg: /var/log/bro).",
empty_allowed=False),
Field(
"bro_bin",
"Bro binary",
"Specify where the Bro binary is located (eg: /opt/bro/bin/bro).",
empty_allowed=False),
Field("bro_opts",
"Bro options",
"Specify options to pass to Bro (None to deactivate).",
empty_allowed=False),
Field("bro_script",
"Bro script",
"Specify a Bro script to use or None do deactivate.",
empty_allowed=False),
Field(
"bro_seeds",
"Bro seed file",
"Specify if you want to use a seed file to predict Bro UIDs or None do deactivate.",
empty_allowed=False),
BooleanField(
"bro_merge",
"Ingest content",
"[Bro 2.1 only] Specify if the extracted content by Bro must be encoded in Base64 and appended to Bro logs. This require a Bro script to be set and this is a True or False option.",
empty_allowed=False),
Field(
"content_maxsize",
"Content maximum size",
"[Bro 2.1 only] Objects greather than the specified size (in bytes) will not be ingested.",
empty_allowed=False),
Field(
"run_maxtime",
"Maximum execution time",
"When a Bro instance run longer than this time (in secs), kill the instance. Set to 0 to deactivate.",
empty_allowed=False),
]
ModularInput.__init__(self, scheme_args, args)
def __init__(self, timeout=30, **kwargs):
scheme_args = {
'title': "FTP",
'description': "Retrieve information over FTP",
'use_single_instance': "false"
}
args = [
IntegerField("port",
"Port",
'The port to run the FTP server on',
none_allowed=False,
empty_allowed=False),
FTPPathField(
"path",
"Path",
'The path to place the received files; relative paths are based on $SPLUNK_HOME',
none_allowed=False,
empty_allowed=False),
Field(
"address",
"Address to Listen on",
'The address to have the FTP server listen on; leave blank to listen on all interfaces',
none_allowed=True,
empty_allowed=True),
#DurationField("interval", "Interval", "The interval defining how often to make sure the server is running", empty_allowed=True, none_allowed=True)
]
ModularInput.__init__(self,
scheme_args,
args,
logger_name="ftp_modular_input")
self.ftp_daemons = []
def __init__(self, timeout=30):
scheme_args = {
'title': "Internet Connection Speedtest",
'description': "A speedtest of the Internet connection",
'use_single_instance': False
}
args = [
Field(
"server",
"Server",
"The server to use for testing; will be automatically assigned if left blank",
empty_allowed=True,
none_allowed=True,
required_on_create=False,
required_on_edit=False),
IntegerField("runs",
"Runs",
"The number of runs that should be executed",
empty_allowed=False,
none_allowed=False)
]
ModularInput.__init__(self,
scheme_args,
args,
logger_name='speedtest_modular_input')
def __init__(self):
scheme_args = {'title': "Syndication Feed (RSS, ATOM, RDF)",
'description': "Import syndication feeds (RSS, ATOM, RDF)",
'use_external_validation': "true",
'streaming_mode': "xml",
'use_single_instance': "true"}
args = [
URLField("url", "Feed URL", "The URL of the feed to input", empty_allowed=False),
BooleanField("include_only_changed", "Include only new or changed entries", "Only include entries that has not been indexed yet (won't get items that were already observed)", empty_allowed=False),
Field("username", "Username", "The username to use for authenticating (only HTTP authentication supported)", none_allowed=True, empty_allowed=True, required_on_create=False, required_on_edit=False),
Field("password", "Password", "The password to use for authenticating (only HTTP authentication supported)", none_allowed=True, empty_allowed=True, required_on_create=False, required_on_edit=False),
DurationField("interval", "Interval", "The interval defining how often to import the feed; can include time units (e.g. 15m for 15 minutes, 8h for 8 hours)", empty_allowed=False),
BooleanField("clean_html", "Convert HTML to Text", "Convert HTML to human readable text", empty_allowed=False)
]
ModularInput.__init__( self, scheme_args, args, logger_name='syndication_modular_input' )
def to_python(self, value, session_key=None):
Field.to_python(self, value, session_key)
# Resolve the path
resolved_path = os.path.normpath(
os.path.join(os.environ['SPLUNK_HOME'], value))
# This is a list of the paths that we will not allow serving.
restricted_paths = []
# The mongo key lives here
restricted_paths.extend(
self.resolve_intermediate_paths('var/lib/splunk/kvstore/mongo'))
# The Splunk secret and certificates live here and the passwd file lives in etc
restricted_paths.extend(self.resolve_intermediate_paths('etc/auth'))
# Make sure that user isn't serving one of the paths that is restricted
if resolved_path in restricted_paths:
raise FieldValidationException(
'The path to serve is not allowed for security' +
'reasons; Splunk paths containing password files, ' +
'certificates, etc. are not allowed to be served')
# Make the path if necessary
try:
os.mkdir(resolved_path)
except OSError:
pass # Directory likely already exists
# Ensure that the path exists
if not os.path.exists(resolved_path):
raise FieldValidationException(
'The path to serve does not exist and could not be' +
'created')
# Ensure that the path is a directory
if not os.path.isdir(resolved_path):
raise FieldValidationException(
'The path to serve is a file, not a directory')
# Return the path that is normalized and resolved
return resolved_path
def to_python(self, value, session_key=None):
Field.to_python(self, value, session_key=None)
# Parse the size
m = DataSizeField.DATA_SIZE_RE.match(value)
# Make sure the duration could be parsed
if m is None:
raise FieldValidationException(
"The value of '%s' for the '%s' parameter is not a valid size of data"
% (str(value), self.name))
# Get the units and duration
d = m.groupdict()
units = d['units'].lower()
# Parse the value provided
try:
size = int(d['size'])
except ValueError:
raise FieldValidationException(
"The size '%s' for the '%s' parameter is not a valid number" %
(d['duration'], self.name))
# Make sure the units are valid
if len(units) > 0 and units not in DataSizeField.UNITS:
raise FieldValidationException(
"The unit '%s' for the '%s' parameter is not a valid unit of duration"
% (units, self.name))
# Convert the units to seconds
if len(units) > 0:
return size * DataSizeField.UNITS[units]
else:
return size
def to_python(self, value, session_key=None):
Field.to_python(self, value, session_key=None)
return value
def to_python(self, value, session_key=None):
Field.to_python(self, value, session_key)
return SelectorField.parse_selector(value, self.name)
def to_python(self, value, session_key=None):
Field.to_python(self, value, session_key=None)
return value
def to_python(self, value):
Field.to_python(self, value)
return value
def to_python(self, value, session_key=None):
Field.to_python(self, value, session_key)
return SelectorField.parse_selector(value, self.name)
def __init__(self, timeout=30, **kwargs):
scheme_args = {
'title': "Web-pages",
'description': "Retrieve information from web-pages",
'use_external_validation': "true",
'streaming_mode': "xml",
'use_single_instance': "true"
}
args = [
Field("title",
"Title",
"A short description (typically just the domain name)",
empty_allowed=False),
URLField(
"url",
"URL",
"The URL to connect to (must be be either HTTP or HTTPS protocol)",
empty_allowed=False,
require_https_on_cloud=True),
DurationField(
"interval",
"Interval",
"The interval defining how often to perform the check; can include time units (e.g. 15m for 15 minutes, 8h for 8 hours)",
empty_allowed=False),
IntegerField("timeout",
"Timeout",
'The timeout (in number of seconds)',
none_allowed=True,
empty_allowed=True),
SelectorField(
"selector",
"Selector",
"A selector that will match the data you want to retrieve",
none_allowed=True,
empty_allowed=True),
# HTTP client options
Field("user_agent",
"User Agent",
"The user-agent to use when communicating with the server",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False),
Field("browser",
"Browser",
'The browser to use',
none_allowed=True,
empty_allowed=True),
# Output options
ListField("name_attributes",
"Field Name Attributes",
"A list of attributes to use for assigning a field name",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False),
BooleanField("use_element_name",
"Use Element Name as Field Name",
"Use the element's tag name as the field name",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False),
BooleanField("output_as_mv",
"Output as Multi-value Field",
"Output the matches as multi-value field",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False),
StaticListField("output_results",
"Indicates when results output should be created",
"Output the matches only when results changed",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False,
valid_values=WebInput.OUTPUT_RESULTS_OPTIONS),
BooleanField("raw_content",
"Raw content",
"Return the raw content returned by the server",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False),
BooleanField("empty_matches",
"Empty matches",
"Include empty rows (otherwise, they are excluded)",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False),
Field(
"text_separator",
"Text Separator",
'A string that will be placed between the extracted values (e.g. a separator of ":" for a match against "<a>tree</a><a>frog</a>" would return "tree:frog")',
none_allowed=True,
empty_allowed=True),
# Spidering options
IntegerField(
"page_limit",
"Discovered page limit",
"A limit on the number of pages that will be auto-discovered",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False),
IntegerField(
"depth_limit",
"Depth limit",
"A limit on how many levels deep the search for pages will go",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False),
Field(
"url_filter",
"URL Filter",
"A wild-card that will indicate which pages it should search for matches in",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False),
# Authentication options
Field("username",
"Username",
"The username to use for authenticating",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False),
Field("password",
"Password",
"The password to use for authenticating",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False),
Field("username_field",
"Username field",
"The name of the username field on the login form",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False),
Field("password_field",
"Password field",
"The name of the password field on the login form",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False),
URLField("authentication_url",
"Authentication URL",
"The URL of the login form",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False,
require_https_on_cloud=True)
]
ModularInput.__init__(self,
scheme_args,
args,
logger_name='web_input_modular_input',
logger_level=logging.INFO)
if timeout > 0:
self.timeout = timeout
else:
self.timeout = 30