def test_url_field_invalid(self):
"""
Ensure that invalid URL fields are detected.
"""
duration_field = DurationField("test_url_field_invalid", "title", "this is a test")
self.assertRaises(FieldValidationException, lambda: duration_field.to_python("1 treefrog"))
self.assertRaises(FieldValidationException, lambda: duration_field.to_python("minute"))
def test_url_field_invalid(self):
"""
Ensure that invalid URL fields are detected.
"""
duration_field = DurationField("test_url_field_invalid", "title",
"this is a test")
self.assertRaises(FieldValidationException,
lambda: duration_field.to_python("1 treefrog"))
self.assertRaises(FieldValidationException,
lambda: duration_field.to_python("minute"))
def __init__(self, timeout=30):
scheme_args = {'title': "Website Availability Check",
'description': "Connects to a website in order to obtain performance statistics",
'use_external_validation': "true",
'streaming_mode': "xml",
'use_single_instance': "true"}
args = [
Field("title", "Title", "A short description (typically just the domain name)", empty_allowed=False),
URLField("url", "URL", "The URL to connect to (must be be either HTTP or HTTPS protocol)", empty_allowed=False),
DurationField("interval", "Interval", "The interval defining how often to perform the check; can include time units (e.g. 15m for 15 minutes, 8h for 8 hours)", empty_allowed=False),
Field("configuration", "Configuration", "Defines a specific proxy configuration to use (in website_monitoring.spec) if not using the default; only used if you want to have multiple proxy servers", none_allowed=True, empty_allowed=True),
Field("client_certificate", "Client Certificate Path", "Defines the path to the client certificate (if the website requires client SSL authentication)", none_allowed=True, empty_allowed=True),
Field("client_certificate_key", "Client Certificate Key Path", "Defines the path to the client certificate key (necessary of the key is in a separate file from the certificate)", none_allowed=True, empty_allowed=True),
Field("username", "Username", "The username to use for authenticating (only HTTP authentication supported)", none_allowed=True, empty_allowed=True, required_on_create=False, required_on_edit=False),
Field("password", "Password", "The password to use for authenticating (only HTTP authentication supported)", none_allowed=True, empty_allowed=True, required_on_create=False, required_on_edit=False)
]
ModularInput.__init__( self, scheme_args, args, logger_name='web_availability_modular_input' )
if timeout > 0:
self.timeout = timeout
else:
self.timeout = 30
def __init__(self):
scheme_args = {
'title': "Google Spreadsheet",
'description':
"Allows you to import/export Splunk lookups to/from Google spreadsheets",
'use_external_validation': "true",
'streaming_mode': "xml",
'use_single_instance': "true"
}
args = [
Field("spreadsheet",
"Spreadsheet Title",
"The title of the spreadsheet",
empty_allowed=False),
Field("worksheet",
"Worksheet Name",
'The name of the worksheet (e.g. "Sheet1")',
empty_allowed=False),
Field(
"service_account_key_file",
"OAuth2 Service Account Key File",
'The service account key with the credentials necessary for authenticating to Google',
empty_allowed=False,
required_on_create=False,
required_on_edit=False),
BooleanField(
"only_if_changed",
"Import file only if changed",
"If set to true, then the import will only be done if the Google worksheet was changed.",
empty_allowed=True,
required_on_create=False,
required_on_edit=False),
Field(
"operation",
"Operation",
"The operation to perform (import into Splunk or export to Google Drive)",
empty_allowed=False),
Field("lookup_name",
"Lookup File Name",
'The name of the lookup file to import the content into',
empty_allowed=False),
DurationField(
"interval",
"Interval",
"The interval defining how often to import the file; can include time units (e.g. 15m for 15 minutes, 8h for 8 hours)",
empty_allowed=False),
DeprecatedField("google_login", "Google Login",
'The login to use when authenticating to Google'),
DeprecatedField(
"google_password", "Google Password",
'The password to use when authenticating to Google. You will need to use an app-specific password here if you are using two-factor authentication.'
)
]
ModularInput.__init__(self,
scheme_args,
args,
logger_name='google_spreadsheet_modular_input')
def __init__(self):
scheme_args = {
'title': "File Meta-data",
'description':
"Import file and directory meta-data (size, modification dates, etc.)",
'use_external_validation': "true",
'streaming_mode': "xml",
'use_single_instance': "true"
}
args = [
FilePathField("file_path",
"File or directory path",
"The path of the to get information on",
empty_allowed=False),
BooleanField(
"recurse",
"Recursively iterate sub-directories",
"Indicates whether sub-directories ought to be recursed",
empty_allowed=False),
BooleanField(
"only_if_changed",
"Changed items only",
"Only include items when one of the time fields is changed",
empty_allowed=False),
DurationField(
"interval",
"Interval",
"The interval defining how often to import the feed; can include time units (e.g. 15m for 15 minutes, 8h for 8 hours)",
empty_allowed=False),
BooleanField("include_file_hash",
"Compute file hash",
"Compute a hash on the file (SHA224)",
empty_allowed=False),
DataSizeField(
"file_hash_limit",
"File-size hash limit",
"Only include items when one of the time fields is changed",
empty_allowed=False),
IntegerField(
"depth_limit",
"Depth Limit",
"A limit on how many directories deep to get results for",
none_allowed=True,
empty_allowed=True),
WildcardField("file_filter",
"File Name Filter",
"A wildcard for which files will be included",
none_allowed=True,
empty_allowed=True)
]
ModularInput.__init__(self,
scheme_args,
args,
logger_name='file_meta_data_modular_input')
def __init__(self, thread_limit=None):
scheme_args = {
'title': "Ping",
'description': "Ping a host or a network to see if it is online",
'use_single_instance': True
}
args = [
ListField("dest",
"Destination",
"The list of hosts or networks to ping",
empty_allowed=True,
none_allowed=True,
required_on_create=True,
required_on_edit=True,
instance_class=DomainOrIPNetworkField),
RangeField("port",
"Port",
"The TCP port to use (leave blank to use ICMP instead)",
low=1,
high=65535,
empty_allowed=True,
none_allowed=True,
required_on_create=False,
required_on_edit=False),
RangeField("runs",
"Runs",
"The number of runs that should be executed",
low=1,
high=100,
empty_allowed=False,
none_allowed=False),
DurationField(
"interval",
"Interval",
"The interval defining how often to perform the check; can include time units (e.g. 15m for 15 minutes, 8h for 8 hours)",
empty_allowed=False)
]
ModularInput.__init__(self,
scheme_args,
args,
logger_name='ping_modular_input',
logger_level=logging.INFO)
if thread_limit is None:
self.thread_limit = PingInput.DEFAULT_THREAD_LIMIT
else:
self.thread_limit = thread_limit
self.threads = {}
def test_duration_valid(self):
"""
Tests the conversion of the duration string to an integer.
"""
duration_field = DurationField("test_duration_valid", "title",
"this is a test")
self.assertEqual(duration_field.to_python("1m"), 60)
self.assertEqual(duration_field.to_python("5m"), 300)
self.assertEqual(duration_field.to_python("5 minute"), 300)
self.assertEqual(duration_field.to_python("5"), 5)
self.assertEqual(duration_field.to_python("5h"), 18000)
self.assertEqual(duration_field.to_python("2d"), 172800)
self.assertEqual(duration_field.to_python("2w"), 86400 * 7 * 2)
def __init__(self):
scheme_args = {'title': "Syndication Feed (RSS, ATOM, RDF)",
'description': "Import syndication feeds (RSS, ATOM, RDF)",
'use_external_validation': "true",
'streaming_mode': "xml",
'use_single_instance': "true"}
args = [
URLField("url", "Feed URL", "The URL of the feed to input", empty_allowed=False),
BooleanField("include_only_changed", "Include only new or changed entries", "Only include entries that has not been indexed yet (won't get items that were already observed)", empty_allowed=False),
Field("username", "Username", "The username to use for authenticating (only HTTP authentication supported)", none_allowed=True, empty_allowed=True, required_on_create=False, required_on_edit=False),
Field("password", "Password", "The password to use for authenticating (only HTTP authentication supported)", none_allowed=True, empty_allowed=True, required_on_create=False, required_on_edit=False),
DurationField("interval", "Interval", "The interval defining how often to import the feed; can include time units (e.g. 15m for 15 minutes, 8h for 8 hours)", empty_allowed=False),
BooleanField("clean_html", "Convert HTML to Text", "Convert HTML to human readable text", empty_allowed=False)
]
ModularInput.__init__( self, scheme_args, args, logger_name='syndication_modular_input' )
def test_duration_valid(self):
"""
Tests the conversion of the duration string to an integer.
"""
duration_field = DurationField("test_duration_valid", "title", "this is a test")
self.assertEqual(duration_field.to_python("1m"), 60)
self.assertEqual(duration_field.to_python("5m"), 300)
self.assertEqual(duration_field.to_python("5 minute"), 300)
self.assertEqual(duration_field.to_python("5"), 5)
self.assertEqual(duration_field.to_python("5h"), 18000)
self.assertEqual(duration_field.to_python("2d"), 172800)
self.assertEqual(duration_field.to_python("2w"), 86400 * 7 * 2)
def __init__(self, thread_limit=None):
scheme_args = {'title': "Port Scan",
'description': "Port scan a host to see what ports are open",
'use_single_instance': True}
args = [
ListField("dest", "Destination", "The list of hosts or networks to port scan", empty_allowed=True, none_allowed=True, required_on_create=True, required_on_edit=True, instance_class=DomainOrIPNetworkField),
PortRangeField("ports", "Ports", "The TCP ports to scan", empty_allowed=False, none_allowed=False, required_on_create=True, required_on_edit=True),
DurationField("interval", "Interval", "The interval defining how often to perform the check; can include time units (e.g. 15m for 15 minutes, 8h for 8 hours)", empty_allowed=False)
]
ModularInput.__init__(self, scheme_args, args, logger_name='portscan_modular_input', logger_level=logging.DEBUG)
if thread_limit is None:
self.thread_limit = PortScanInput.DEFAULT_THREAD_LIMIT
else:
self.thread_limit = thread_limit
self.threads = {}
def __init__(self, timeout=30, **kwargs):
scheme_args = {
'title': "Web-pages",
'description': "Retrieve information from web-pages",
'use_external_validation': "true",
'streaming_mode': "xml",
'use_single_instance': "true"
}
args = [
Field("title",
"Title",
"A short description (typically just the domain name)",
empty_allowed=False),
URLField(
"url",
"URL",
"The URL to connect to (must be be either HTTP or HTTPS protocol)",
empty_allowed=False,
require_https_on_cloud=True),
DurationField(
"interval",
"Interval",
"The interval defining how often to perform the check; can include time units (e.g. 15m for 15 minutes, 8h for 8 hours)",
empty_allowed=False),
IntegerField("timeout",
"Timeout",
'The timeout (in number of seconds)',
none_allowed=True,
empty_allowed=True),
SelectorField(
"selector",
"Selector",
"A selector that will match the data you want to retrieve",
none_allowed=True,
empty_allowed=True),
# HTTP client options
Field("user_agent",
"User Agent",
"The user-agent to use when communicating with the server",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False),
Field("browser",
"Browser",
'The browser to use',
none_allowed=True,
empty_allowed=True),
# Output options
ListField("name_attributes",
"Field Name Attributes",
"A list of attributes to use for assigning a field name",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False),
BooleanField("use_element_name",
"Use Element Name as Field Name",
"Use the element's tag name as the field name",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False),
BooleanField("output_as_mv",
"Output as Multi-value Field",
"Output the matches as multi-value field",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False),
StaticListField("output_results",
"Indicates when results output should be created",
"Output the matches only when results changed",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False,
valid_values=WebInput.OUTPUT_RESULTS_OPTIONS),
BooleanField("raw_content",
"Raw content",
"Return the raw content returned by the server",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False),
BooleanField("empty_matches",
"Empty matches",
"Include empty rows (otherwise, they are excluded)",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False),
Field(
"text_separator",
"Text Separator",
'A string that will be placed between the extracted values (e.g. a separator of ":" for a match against "<a>tree</a><a>frog</a>" would return "tree:frog")',
none_allowed=True,
empty_allowed=True),
# Spidering options
IntegerField(
"page_limit",
"Discovered page limit",
"A limit on the number of pages that will be auto-discovered",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False),
IntegerField(
"depth_limit",
"Depth limit",
"A limit on how many levels deep the search for pages will go",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False),
Field(
"url_filter",
"URL Filter",
"A wild-card that will indicate which pages it should search for matches in",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False),
# Authentication options
Field("username",
"Username",
"The username to use for authenticating",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False),
Field("password",
"Password",
"The password to use for authenticating",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False),
Field("username_field",
"Username field",
"The name of the username field on the login form",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False),
Field("password_field",
"Password field",
"The name of the password field on the login form",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False),
URLField("authentication_url",
"Authentication URL",
"The URL of the login form",
none_allowed=True,
empty_allowed=True,
required_on_create=False,
required_on_edit=False,
require_https_on_cloud=True)
]
ModularInput.__init__(self,
scheme_args,
args,
logger_name='web_input_modular_input',
logger_level=logging.INFO)
if timeout > 0:
self.timeout = timeout
else:
self.timeout = 30