def __init__(self):
scheme_args = {
'title': "Threat List Manager",
'description':
"Merges threatlist information into Splunk lookup tables.",
'use_external_validation': "true",
'streaming_mode': "xml",
'use_single_instance': "true"
}
args = [
Field("master_host",
"Master host",
"The master host for this download.",
required_on_create=False,
required_on_edit=False)
]
self._app = 'SA-ThreatIntelligence'
self._name = 'ThreatlistManager'
self._owner = 'nobody'
# The alternate modular input name from which to retrieve stanza information.
self._alt_modinput_name = 'threatlist'
self._logger = setup_logger(name='threatlist_manager',
level=logging.INFO)
super(ThreatlistManagerModularInput, self).__init__(scheme_args, args)
def __init__(self):
scheme_args = {
'title': "Facebook Threat Exchange",
'description':
"Enables consumption of Threat Intelligence from the Facebook Threat Exchange",
'use_external_validation': "true",
'streaming_mode': "json",
'use_single_instance': "true"
}
args = [
#General Options
Field(
"type", "IndicatorType",
"The IndicatorType to collect. Leaving blank collects all IndicatorTypes.",
False, False),
Field(
"app_id", "App-ID",
"Threat Exchange App-ID (Note: app_secret must be kept in Credential Manager.",
True, True),
Field(
"since", "Since",
"A Unix timestamp or PHP-style strtotime data value that points to the start of the range of time-based data.",
True, True),
BooleanField(
"include_expired", "Include Expired?",
"When set to true, expired intel will also be collected."),
RangeField("limit", "Limit",
"Maximum number of results per API request. (1-1000)",
1, 1000, True, True),
IntegerField(
"request_limit", "API Request Limit",
"Maximum number of subsequent API requests per stanza per Modular Input execution.",
True, True),
RangeField(
"max_confidence", "Max Confidence",
"Maximum allowed confidence value for the intel returned. (0 - 100).",
0, 100, True, True),
RangeField(
"min_confidence", "Min Confidence",
"Minimum allowed confidence value for the intel returned. (0 - 100).",
0, 100, True, True)
]
self._app = self.APP
self._owner = self.OWNER
self._name = 'Facebook Threat Exchange'
self._logger = setup_logger(name='fb_threat_exchange',
level=logging.INFO)
super(FacebookThreatExchange, self).__init__(scheme_args, args)
import os, msvcrt
msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY)
import splunk.admin as admin
import splunk.bundle as bundle
import splunk.entity as entity
import splunk.util as util
from splunk import ResourceNotFound
from splunk.clilib.bundle_paths import make_splunkhome_path
sys.path.append(make_splunkhome_path(['etc', 'apps', 'SA-Utils', 'lib']))
from SolnCommon.log import setup_logger, SHORT_FORMAT
logger = setup_logger('log_review_rest_handler', level=logging.INFO, format=SHORT_FORMAT)
class InvalidConfigException(Exception):
"""
Describes an invalid configuration.
"""
pass
class InvalidParameterValueException(InvalidConfigException):
"""
Describes a config parameter that has an invalid value.
"""
def __init__(self, field, value, value_must_be):
message = "The value for the field '%s' is invalid: %s (was %s)" % (field, value_must_be, value)
from SolnCommon.modinput import Field
from SolnCommon.modinput import IntegerField
from SolnCommon.modinput import ModularInput
from SolnCommon.pathutils import construct_os_path, expandvars_restricted
from SolnCommon.pooling import should_execute
sys.path.append(
make_splunkhome_path(["etc", "apps", "SA-ThreatIntelligence", "contrib"]))
from parsers.parser_exceptions import ParserConfigurationException, ParserEmptyException, ParserException
from parsers.ioc_parser import IOCParser
from parsers.csv_parser import CSVParser
from parsers.utils import IntelUtils, LookupUtils, ParserUtils
from parsers.stix_parser import STIXParser
logger = setup_logger(name='threat_intelligence_manager', level=logging.INFO)
# Reroute warnings from python-stix
logging.captureWarnings(True)
warn_logger = logging.getLogger('py.warnings')
warn_logger.propagate = False
warn_logger.handlers = [logger.handlers[0]]
class ThreatIntelMeta(object):
def __init__(self, app, collection, kv, owner):
'''Initialize a class for handling of threat intel metadata.
Arguments:
app - The app where the metadata collection is housed.
collection - The name of the collection.
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY)
import splunk.admin as admin
import splunk.entity as entity
import splunk.rest as rest
import splunk.util as util
import time
from splunk import ResourceNotFound
from splunk.clilib.bundle_paths import make_splunkhome_path
sys.path.append(make_splunkhome_path(['etc', 'apps', 'SA-Utils', 'lib']))
from SolnCommon.log import setup_logger, SHORT_FORMAT
logger = setup_logger('transitioners_rest_handler',
level=logging.INFO,
format=SHORT_FORMAT)
def time_function_call(fx):
"""
This decorator will provide a log message measuring how long a function call took.
Arguments:
fx -- The function to measure
"""
def wrapper(*args, **kwargs):
t = time.time()
r = fx(*args, **kwargs)
import splunk.search
import splunk.util
from splunk.clilib.bundle_paths import make_splunkhome_path
sys.path.append(make_splunkhome_path(["etc", "apps", "SA-Utils", "lib"]))
sys.path.append(
make_splunkhome_path(["etc", "apps", "SA-IdentityManagement", "lib"]))
from SolnCommon.log import setup_logger
from SolnCommon.lookup_conversion.lookup_modinput import LookupModularInput
from SolnCommon.metadata import MetadataReader
from SolnCommon.modinput.fields import Field
from identity_generation import get_conventions, generate_search_string
from identity_macros import IdentityCorrelationMacro
from identity_sources import generate_identity_source
import logging
logger = setup_logger('identity_manager', level=logging.INFO)
class IdentityManagerModularInput(LookupModularInput):
MACRO_AUTOUPDATE_PERMITTED = 'enable_identity_management_autoupdate'
MACRO_TIMEOUT = 'identity_management_timeout'
CHECKPOINT_PREFIX = 'identityLookup_conf'
DEFAULT_TIMEOUT = 30
def __init__(self):
scheme_args = {
'title': "Identity Management",
'description':
"Merges asset and identity information into Splunk lookup tables.",
if sys.platform == "win32":
import os, msvcrt
msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY)
import splunk
import splunk.admin
import splunk.rest
from splunk.clilib.bundle_paths import make_splunkhome_path
sys.path.append(make_splunkhome_path(['etc', 'apps', 'SA-Utils', 'lib']))
from SolnCommon.log import setup_logger, SHORT_FORMAT
logger = setup_logger('identity_correlation_rest_handler',
level=logging.INFO,
format=SHORT_FORMAT)
class AutomaticLookup(object):
"""Wrapper class for data/props/lookups, used in creation of automatic lookups."""
LOOKUP_URI_BASE = '/servicesNS/{owner}/{namespace}/data/props/lookups'
LOOKUP_URI_FULL = '/servicesNS/{owner}/{namespace}/data/props/lookups/{name}'
@staticmethod
def list(owner, namespace, session_key):
"""
List automatic lookups.
:param owner: A Splunk user.
import os, msvcrt
msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY)
import splunk.admin as admin
import splunk.entity as entity
import splunk.util as util
from notable_event_suppression import NotableEventSuppression
from splunk import ResourceNotFound
from splunk.clilib.bundle_paths import make_splunkhome_path
sys.path.append(make_splunkhome_path(["etc", "apps", "SA-Utils", "lib"]))
from SolnCommon.log import setup_logger, SHORT_FORMAT
logger = setup_logger('suppressions_rest_handler', format=SHORT_FORMAT)
logger.setLevel(logging.INFO)
class InvalidConfigException(Exception):
pass
class InvalidParameterValueException(InvalidConfigException):
"""
Describes a config parameter that has an invalid value.
"""
def __init__(self, field, value, value_must_be):
message = "The value for the parameter '%s' is invalid: %s (was %s)" % (
field, value_must_be, value)
super(InvalidConfigException, self).__init__(message)
if sys.platform == "win32":
import os, msvcrt
msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY)
from splunk import ResourceNotFound
from splunk.clilib.bundle_paths import make_splunkhome_path
sys.path.append(make_splunkhome_path(['etc', 'apps', 'SA-Utils', 'lib']))
from SolnCommon.log import setup_logger, SHORT_FORMAT
from SolnCommon.kvstore import KvStoreHandler
from SolnCommon.pooling import should_execute
logger = setup_logger('governance_rest_handler',
level=logging.INFO,
format=SHORT_FORMAT)
class UnauthorizedUserException(Exception):
pass
class InvalidConfigException(Exception):
pass
class InvalidParameterValueException(InvalidConfigException):
"""
Describes a config parameter that has an invalid value.
"""
import splunk.admin as admin
import splunk.entity as entity
import splunk.rest as rest
import splunk.util as util
from splunk import ResourceNotFound
from shortcuts import NotableOwner
from splunk.clilib.bundle_paths import make_splunkhome_path
sys.path.append(make_splunkhome_path(['etc', 'apps', 'SA-Utils', 'lib']))
from SolnCommon.log import setup_logger, SHORT_FORMAT
from SolnCommon.kvstore import KvStoreHandler
from SolnCommon.pooling import should_execute
logger = setup_logger('correlationsearches_rest_handler',
level=logging.INFO,
format=SHORT_FORMAT)
class UnauthorizedUserException(Exception):
pass
class InvalidConfigException(Exception):
pass
class InvalidParameterValueException(InvalidConfigException):
"""
Describes a config parameter that has an invalid value.
"""
def __init__(self):
self.DEFAULT_INITIAL_DELAY = 300
self.DEFAULT_RETRIES = 3
self.DEFAULT_RETRY_INTERVAL = 60
self.DEFAULT_TIMEOUT_INTERVAL = 30
self.DEFAULT_SKIP_HEADER_LINES = 0
self.DEFAULT_THREAD_POOL_SIZE = 5
self.DEFAULT_THREAD_SLEEP_INTERVAL = 300
self.DEFAULT_MERGE_THREAD_SLEEP_INTERVAL = 15
self.HANDLER_HTTP = 'http'
self.HANDLER_HTTPS = 'https'
self.HANDLER_LOOKUP = 'lookup'
self.HANDLER_TAXII = 'taxii'
# Dictionary of supported protocol handlers.
self.PROTOCOL_HANDLERS = {
self.HANDLER_HTTP: HttpProtocolHandler,
self.HANDLER_HTTPS: HttpProtocolHandler,
self.HANDLER_LOOKUP: NoopProtocolHandler,
self.HANDLER_TAXII: TaxiiHandler
}
# Regex for extracting key=value strings
self.KV_REGEX = re.compile(r'(\w+)=([\w:$]+|"[^"]+")')
# Regex for extracting interpolated arguments.
self.ARG_REGEX = re.compile(r'\$([A-Za-z0-9_]+):([A-Za-z0-9_]+)\$')
# Default target directory
self.THREAT_INTEL_TARGET_PATH = make_splunkhome_path([
'etc', 'apps', 'SA-ThreatIntelligence', 'local', 'data',
'threat_intel'
])
# Default exclusions - these are the types of threatlist that don't get
# written to self.THREAT_INTEL_TARGET_PATH
self.THREAT_INTEL_EXCLUSIONS = ['alexa', 'asn', 'mozilla_psl', 'tld']
self.DEPRECATED_STANZAS = [
'maxmind_geoip_asn_ipv4', 'maxmind_geoip_asn_ipv6'
]
scheme_args = {
'title': "Threat Intelligence Downloads",
'description':
"Downloads threat lists or other threat intelligence feeds from remote hosts.",
'use_external_validation': "true",
'streaming_mode': "xml",
'use_single_instance': "false"
}
args = [
# General options
Field(
"type",
"Threatlist Type",
"""Type of threat list, such as "malware". Must be "taxii" for TAXII feeds.""",
required_on_create=True,
required_on_edit=True),
Field("description",
"Description",
"""Description of the threat list.""",
required_on_create=True,
required_on_edit=True),
Field(
"max_age",
"Maximum age",
"Maximum age for threat content (provided for use by consumers of threat content)",
required_on_create=False,
required_on_edit=False),
Field("target",
"Target",
"""Target lookup table.""",
required_on_create=False,
required_on_edit=False),
Field("url",
"URL",
"""URL or location of the threatlist.""",
required_on_create=True,
required_on_edit=True),
RangeField(
"weight",
"Weight",
"""Weight for IPs that appear on this threatlist. A higher weight increases an IP's risk score.""",
low=1,
high=100,
required_on_create=True,
required_on_edit=True),
# Download options
Field("post_args",
"POST arguments",
"""POST arguments to send to the remote URL.""",
required_on_create=False,
required_on_edit=False),
IntegerField(
"retries",
"Retries",
"the number of times to retry a failed download. [Defaults to {0}]"
.format(self.DEFAULT_RETRIES),
required_on_create=True,
required_on_edit=True),
IntegerField(
"retry_interval",
"Retry interval",
"Interval between attempts to download this threat list, in seconds. [Defaults to {0}]"
.format(self.DEFAULT_RETRY_INTERVAL),
required_on_create=True,
required_on_edit=True),
Field(
"site_user",
"Remote site user",
"The user name for authentication to the remote site, if required. Must correspond to a Splunk stored credential.",
required_on_create=False,
required_on_edit=False),
IntegerField(
"timeout",
"Timeout interval",
"Time before regarding a download attempt as failed, in seconds. [Defaults to {0}]"
.format(self.DEFAULT_TIMEOUT_INTERVAL),
required_on_create=True,
required_on_edit=True),
# Proxy options
RangeField("proxy_port",
"Proxy port",
"The proxy server port, if required.",
low=0,
high=65535,
required_on_create=False,
required_on_edit=False),
Field(
"proxy_server",
"Proxy server",
"The proxy server, if required. Only used by HTTP(S) protocol.",
required_on_create=False,
required_on_edit=False),
Field(
"proxy_user",
"Proxy user",
"The proxy user name, if required. Must correspond to a Splunk stored credential. Only used by HTTP(s) protocol.",
required_on_create=False,
required_on_edit=False),
# Parser options
Field("delim_regex",
"Delimiting regex",
"Regular expression used to delimit the input.",
required_on_create=False,
required_on_edit=False),
Field("extract_regex",
"Extracting regex",
"Regular expression used to extract fields from the input.",
required_on_create=False,
required_on_edit=False),
Field("fields",
"Fields",
"The list of fields to extract from the threat list.",
required_on_create=False,
required_on_edit=False),
Field(
"ignore_regex",
"Ignoring regex",
"Regular expression for lines to be ignored in the threat list.",
required_on_create=False,
required_on_edit=False),
Field("skip_header_lines",
"Skip header lines",
"Number of header lines to skip, if any. [Defaults to {0}]".
format(self.DEFAULT_SKIP_HEADER_LINES),
required_on_create=False,
required_on_edit=False),
# General processing options - should only be set in default stanza.
IntegerField(
"initial_delay",
"Initial delay",
"""Initial delay in seconds before the modular input begins executing, IF not being executed on a cron schedule. Used to alleviate startup load. [Defaults to {0}]"""
.format(self.DEFAULT_INITIAL_DELAY),
required_on_create=False,
required_on_edit=False),
Field("master_host",
"Master host",
"The master host for this download.",
required_on_create=False,
required_on_edit=False),
]
self._app = 'SA-ThreatIntelligence'
self._owner = 'nobody'
self._name = 'Threatlist'
self._logger = setup_logger(name='threatlist', level=logging.INFO)
super(ThreatlistModularInput, self).__init__(scheme_args, args)
import collections
import itertools
import logging
import sys
import operator
import json
import httplib
from splunk.persistconn.application import PersistentServerConnectionApplication
from splunk.clilib.bundle_paths import make_splunkhome_path
import splunk.search
sys.path.append(make_splunkhome_path(["etc", "apps", "SA-Utils", "lib"]))
from SolnCommon.log import setup_logger, SHORT_FORMAT
logger = setup_logger('identitymapper', format=SHORT_FORMAT, level=logging.INFO)
if sys.platform == "win32":
import os, msvcrt
msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY)
class IdentityMapperErrors(object):
'''Enum for error strings.'''
ERR_INVALID_ARG = 'Invalid argument provided'
ERR_INVALID_ARGC = 'Invalid argument count provided'
ERR_INVALID_CONSTRAINT_METHOD = 'Invalid constraint method requested'
ERR_INVALID_CONSTRAINT_FIELD = 'Invalid constraint field requested'
class IdentityMapper(PersistentServerConnectionApplication):
from splunk.appserver.mrsparkle.lib.decorators import expose_page
from splunk.appserver.mrsparkle.lib.routes import route
from splunk.rest import simpleRequest
from splunk.clilib.bundle_paths import make_splunkhome_path
sys.path.append(make_splunkhome_path(['etc', 'apps', 'SA-Utils', 'lib']))
from SolnCommon.log import setup_logger, SHORT_FORMAT
# Import the correlation search helper class
sys.path.append(
make_splunkhome_path(["etc", "apps", "SA-ThreatIntelligence", "bin"]))
from correlation_search import CorrelationSearch
from custom_search_builder.base import CustomSearchBuilderBase
from shortcuts import Severity
logger = setup_logger('correlation_search_controller',
level=logging.DEBUG,
format=SHORT_FORMAT)
class CorrelationSearchBuilder(controllers.BaseController):
'''Correlation search builder Controller'''
@route('/:ping=ping')
@expose_page(must_login=True, methods=['GET', 'POST'])
def ping(self, **kwargs):
return self.render_json({'is_available': '1'}, set_mime='text/plain')
@route('/:save=save')
@expose_page(must_login=True, methods=['POST'])
def save(self, **kwargs):
"""
This handler is currently invoked by the "Save correlation search" button that is included in predictive analytics. You should be using the 'update_or_create_search' for new functionality.
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY)
import splunk
import splunk.admin
import splunk.rest
import splunk.util
from splunk import ResourceNotFound
from splunk.clilib.bundle_paths import make_splunkhome_path
from splunk.appserver.mrsparkle.lib import i18n
sys.path.append(make_splunkhome_path(["etc", "apps", "SA-Utils", "lib"]))
from SolnCommon.log import setup_logger, SHORT_FORMAT
logger = setup_logger('identityLookup_rest_handler',
level=logging.INFO,
format=SHORT_FORMAT)
class IdentityLookup(object):
# See definitions for convention keys and values in identityLookup.conf.spec.
CONVENTION_KEY_RX = re.compile(r'^(convention\.(\d+))$')
REQUIRED_PARAMS = [
'exact', 'email', 'email_short', 'convention', 'case_sensitive'
]
OPTIONAL_PARAMS = ['convention.*']
LEGACY_PARAMS = ['match_order']
VALID_PARAMS = REQUIRED_PARAMS + OPTIONAL_PARAMS + LEGACY_PARAMS
sys.path.append(make_splunkhome_path(['etc', 'apps', 'SA-Utils', 'lib']))
from SolnCommon.searchutils import parse_search_string
from SolnCommon.log import setup_logger, SHORT_FORMAT
# Ensure that shortcuts can be imported.
sys.path.append(
make_splunkhome_path(["etc", "apps", "SA-ThreatIntelligence", "bin"]))
from shortcuts import Duration
from shortcuts import Severity
from shortcuts import NotableOwner
# Import the custom search builder
from custom_search_builder.base import CustomSearchBuilderBase
from custom_search_builder.make_correlation_search import makeCorrelationSearch
logger = setup_logger('correlationsearches_base_class',
level=logging.DEBUG,
format=SHORT_FORMAT)
def error(key):
'''
Returns an error message for a given field.
'''
# Dictionary of error messages
error_descriptions = {
'alert.suppress.fields':
"One or more fields must be selected to group by.",
'alert.suppress.period':
"Aggregation window duration must be a positive integer."
}
