def __init__(self): scheme_args = { 'title': "Threat List Manager", 'description': "Merges threatlist information into Splunk lookup tables.", 'use_external_validation': "true", 'streaming_mode': "xml", 'use_single_instance': "true" } args = [ Field("master_host", "Master host", "The master host for this download.", required_on_create=False, required_on_edit=False) ] self._app = 'SA-ThreatIntelligence' self._name = 'ThreatlistManager' self._owner = 'nobody' # The alternate modular input name from which to retrieve stanza information. self._alt_modinput_name = 'threatlist' self._logger = setup_logger(name='threatlist_manager', level=logging.INFO) super(ThreatlistManagerModularInput, self).__init__(scheme_args, args)
def __init__(self): scheme_args = { 'title': "Facebook Threat Exchange", 'description': "Enables consumption of Threat Intelligence from the Facebook Threat Exchange", 'use_external_validation': "true", 'streaming_mode': "json", 'use_single_instance': "true" } args = [ #General Options Field( "type", "IndicatorType", "The IndicatorType to collect. Leaving blank collects all IndicatorTypes.", False, False), Field( "app_id", "App-ID", "Threat Exchange App-ID (Note: app_secret must be kept in Credential Manager.", True, True), Field( "since", "Since", "A Unix timestamp or PHP-style strtotime data value that points to the start of the range of time-based data.", True, True), BooleanField( "include_expired", "Include Expired?", "When set to true, expired intel will also be collected."), RangeField("limit", "Limit", "Maximum number of results per API request. (1-1000)", 1, 1000, True, True), IntegerField( "request_limit", "API Request Limit", "Maximum number of subsequent API requests per stanza per Modular Input execution.", True, True), RangeField( "max_confidence", "Max Confidence", "Maximum allowed confidence value for the intel returned. (0 - 100).", 0, 100, True, True), RangeField( "min_confidence", "Min Confidence", "Minimum allowed confidence value for the intel returned. (0 - 100).", 0, 100, True, True) ] self._app = self.APP self._owner = self.OWNER self._name = 'Facebook Threat Exchange' self._logger = setup_logger(name='fb_threat_exchange', level=logging.INFO) super(FacebookThreatExchange, self).__init__(scheme_args, args)
import os, msvcrt msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY) msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY) import splunk.admin as admin import splunk.bundle as bundle import splunk.entity as entity import splunk.util as util from splunk import ResourceNotFound from splunk.clilib.bundle_paths import make_splunkhome_path sys.path.append(make_splunkhome_path(['etc', 'apps', 'SA-Utils', 'lib'])) from SolnCommon.log import setup_logger, SHORT_FORMAT logger = setup_logger('log_review_rest_handler', level=logging.INFO, format=SHORT_FORMAT) class InvalidConfigException(Exception): """ Describes an invalid configuration. """ pass class InvalidParameterValueException(InvalidConfigException): """ Describes a config parameter that has an invalid value. """ def __init__(self, field, value, value_must_be): message = "The value for the field '%s' is invalid: %s (was %s)" % (field, value_must_be, value)
from SolnCommon.modinput import Field from SolnCommon.modinput import IntegerField from SolnCommon.modinput import ModularInput from SolnCommon.pathutils import construct_os_path, expandvars_restricted from SolnCommon.pooling import should_execute sys.path.append( make_splunkhome_path(["etc", "apps", "SA-ThreatIntelligence", "contrib"])) from parsers.parser_exceptions import ParserConfigurationException, ParserEmptyException, ParserException from parsers.ioc_parser import IOCParser from parsers.csv_parser import CSVParser from parsers.utils import IntelUtils, LookupUtils, ParserUtils from parsers.stix_parser import STIXParser logger = setup_logger(name='threat_intelligence_manager', level=logging.INFO) # Reroute warnings from python-stix logging.captureWarnings(True) warn_logger = logging.getLogger('py.warnings') warn_logger.propagate = False warn_logger.handlers = [logger.handlers[0]] class ThreatIntelMeta(object): def __init__(self, app, collection, kv, owner): '''Initialize a class for handling of threat intel metadata. Arguments: app - The app where the metadata collection is housed. collection - The name of the collection.
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY) import splunk.admin as admin import splunk.entity as entity import splunk.rest as rest import splunk.util as util import time from splunk import ResourceNotFound from splunk.clilib.bundle_paths import make_splunkhome_path sys.path.append(make_splunkhome_path(['etc', 'apps', 'SA-Utils', 'lib'])) from SolnCommon.log import setup_logger, SHORT_FORMAT logger = setup_logger('transitioners_rest_handler', level=logging.INFO, format=SHORT_FORMAT) def time_function_call(fx): """ This decorator will provide a log message measuring how long a function call took. Arguments: fx -- The function to measure """ def wrapper(*args, **kwargs): t = time.time() r = fx(*args, **kwargs)
import splunk.search import splunk.util from splunk.clilib.bundle_paths import make_splunkhome_path sys.path.append(make_splunkhome_path(["etc", "apps", "SA-Utils", "lib"])) sys.path.append( make_splunkhome_path(["etc", "apps", "SA-IdentityManagement", "lib"])) from SolnCommon.log import setup_logger from SolnCommon.lookup_conversion.lookup_modinput import LookupModularInput from SolnCommon.metadata import MetadataReader from SolnCommon.modinput.fields import Field from identity_generation import get_conventions, generate_search_string from identity_macros import IdentityCorrelationMacro from identity_sources import generate_identity_source import logging logger = setup_logger('identity_manager', level=logging.INFO) class IdentityManagerModularInput(LookupModularInput): MACRO_AUTOUPDATE_PERMITTED = 'enable_identity_management_autoupdate' MACRO_TIMEOUT = 'identity_management_timeout' CHECKPOINT_PREFIX = 'identityLookup_conf' DEFAULT_TIMEOUT = 30 def __init__(self): scheme_args = { 'title': "Identity Management", 'description': "Merges asset and identity information into Splunk lookup tables.",
if sys.platform == "win32": import os, msvcrt msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY) msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY) import splunk import splunk.admin import splunk.rest from splunk.clilib.bundle_paths import make_splunkhome_path sys.path.append(make_splunkhome_path(['etc', 'apps', 'SA-Utils', 'lib'])) from SolnCommon.log import setup_logger, SHORT_FORMAT logger = setup_logger('identity_correlation_rest_handler', level=logging.INFO, format=SHORT_FORMAT) class AutomaticLookup(object): """Wrapper class for data/props/lookups, used in creation of automatic lookups.""" LOOKUP_URI_BASE = '/servicesNS/{owner}/{namespace}/data/props/lookups' LOOKUP_URI_FULL = '/servicesNS/{owner}/{namespace}/data/props/lookups/{name}' @staticmethod def list(owner, namespace, session_key): """ List automatic lookups. :param owner: A Splunk user.
import os, msvcrt msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY) msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY) import splunk.admin as admin import splunk.entity as entity import splunk.util as util from notable_event_suppression import NotableEventSuppression from splunk import ResourceNotFound from splunk.clilib.bundle_paths import make_splunkhome_path sys.path.append(make_splunkhome_path(["etc", "apps", "SA-Utils", "lib"])) from SolnCommon.log import setup_logger, SHORT_FORMAT logger = setup_logger('suppressions_rest_handler', format=SHORT_FORMAT) logger.setLevel(logging.INFO) class InvalidConfigException(Exception): pass class InvalidParameterValueException(InvalidConfigException): """ Describes a config parameter that has an invalid value. """ def __init__(self, field, value, value_must_be): message = "The value for the parameter '%s' is invalid: %s (was %s)" % ( field, value_must_be, value) super(InvalidConfigException, self).__init__(message)
if sys.platform == "win32": import os, msvcrt msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY) msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY) from splunk import ResourceNotFound from splunk.clilib.bundle_paths import make_splunkhome_path sys.path.append(make_splunkhome_path(['etc', 'apps', 'SA-Utils', 'lib'])) from SolnCommon.log import setup_logger, SHORT_FORMAT from SolnCommon.kvstore import KvStoreHandler from SolnCommon.pooling import should_execute logger = setup_logger('governance_rest_handler', level=logging.INFO, format=SHORT_FORMAT) class UnauthorizedUserException(Exception): pass class InvalidConfigException(Exception): pass class InvalidParameterValueException(InvalidConfigException): """ Describes a config parameter that has an invalid value. """
import splunk.admin as admin import splunk.entity as entity import splunk.rest as rest import splunk.util as util from splunk import ResourceNotFound from shortcuts import NotableOwner from splunk.clilib.bundle_paths import make_splunkhome_path sys.path.append(make_splunkhome_path(['etc', 'apps', 'SA-Utils', 'lib'])) from SolnCommon.log import setup_logger, SHORT_FORMAT from SolnCommon.kvstore import KvStoreHandler from SolnCommon.pooling import should_execute logger = setup_logger('correlationsearches_rest_handler', level=logging.INFO, format=SHORT_FORMAT) class UnauthorizedUserException(Exception): pass class InvalidConfigException(Exception): pass class InvalidParameterValueException(InvalidConfigException): """ Describes a config parameter that has an invalid value. """
def __init__(self): self.DEFAULT_INITIAL_DELAY = 300 self.DEFAULT_RETRIES = 3 self.DEFAULT_RETRY_INTERVAL = 60 self.DEFAULT_TIMEOUT_INTERVAL = 30 self.DEFAULT_SKIP_HEADER_LINES = 0 self.DEFAULT_THREAD_POOL_SIZE = 5 self.DEFAULT_THREAD_SLEEP_INTERVAL = 300 self.DEFAULT_MERGE_THREAD_SLEEP_INTERVAL = 15 self.HANDLER_HTTP = 'http' self.HANDLER_HTTPS = 'https' self.HANDLER_LOOKUP = 'lookup' self.HANDLER_TAXII = 'taxii' # Dictionary of supported protocol handlers. self.PROTOCOL_HANDLERS = { self.HANDLER_HTTP: HttpProtocolHandler, self.HANDLER_HTTPS: HttpProtocolHandler, self.HANDLER_LOOKUP: NoopProtocolHandler, self.HANDLER_TAXII: TaxiiHandler } # Regex for extracting key=value strings self.KV_REGEX = re.compile(r'(\w+)=([\w:$]+|"[^"]+")') # Regex for extracting interpolated arguments. self.ARG_REGEX = re.compile(r'\$([A-Za-z0-9_]+):([A-Za-z0-9_]+)\$') # Default target directory self.THREAT_INTEL_TARGET_PATH = make_splunkhome_path([ 'etc', 'apps', 'SA-ThreatIntelligence', 'local', 'data', 'threat_intel' ]) # Default exclusions - these are the types of threatlist that don't get # written to self.THREAT_INTEL_TARGET_PATH self.THREAT_INTEL_EXCLUSIONS = ['alexa', 'asn', 'mozilla_psl', 'tld'] self.DEPRECATED_STANZAS = [ 'maxmind_geoip_asn_ipv4', 'maxmind_geoip_asn_ipv6' ] scheme_args = { 'title': "Threat Intelligence Downloads", 'description': "Downloads threat lists or other threat intelligence feeds from remote hosts.", 'use_external_validation': "true", 'streaming_mode': "xml", 'use_single_instance': "false" } args = [ # General options Field( "type", "Threatlist Type", """Type of threat list, such as "malware". Must be "taxii" for TAXII feeds.""", required_on_create=True, required_on_edit=True), Field("description", "Description", """Description of the threat list.""", required_on_create=True, required_on_edit=True), Field( "max_age", "Maximum age", "Maximum age for threat content (provided for use by consumers of threat content)", required_on_create=False, required_on_edit=False), Field("target", "Target", """Target lookup table.""", required_on_create=False, required_on_edit=False), Field("url", "URL", """URL or location of the threatlist.""", required_on_create=True, required_on_edit=True), RangeField( "weight", "Weight", """Weight for IPs that appear on this threatlist. A higher weight increases an IP's risk score.""", low=1, high=100, required_on_create=True, required_on_edit=True), # Download options Field("post_args", "POST arguments", """POST arguments to send to the remote URL.""", required_on_create=False, required_on_edit=False), IntegerField( "retries", "Retries", "the number of times to retry a failed download. [Defaults to {0}]" .format(self.DEFAULT_RETRIES), required_on_create=True, required_on_edit=True), IntegerField( "retry_interval", "Retry interval", "Interval between attempts to download this threat list, in seconds. [Defaults to {0}]" .format(self.DEFAULT_RETRY_INTERVAL), required_on_create=True, required_on_edit=True), Field( "site_user", "Remote site user", "The user name for authentication to the remote site, if required. Must correspond to a Splunk stored credential.", required_on_create=False, required_on_edit=False), IntegerField( "timeout", "Timeout interval", "Time before regarding a download attempt as failed, in seconds. [Defaults to {0}]" .format(self.DEFAULT_TIMEOUT_INTERVAL), required_on_create=True, required_on_edit=True), # Proxy options RangeField("proxy_port", "Proxy port", "The proxy server port, if required.", low=0, high=65535, required_on_create=False, required_on_edit=False), Field( "proxy_server", "Proxy server", "The proxy server, if required. Only used by HTTP(S) protocol.", required_on_create=False, required_on_edit=False), Field( "proxy_user", "Proxy user", "The proxy user name, if required. Must correspond to a Splunk stored credential. Only used by HTTP(s) protocol.", required_on_create=False, required_on_edit=False), # Parser options Field("delim_regex", "Delimiting regex", "Regular expression used to delimit the input.", required_on_create=False, required_on_edit=False), Field("extract_regex", "Extracting regex", "Regular expression used to extract fields from the input.", required_on_create=False, required_on_edit=False), Field("fields", "Fields", "The list of fields to extract from the threat list.", required_on_create=False, required_on_edit=False), Field( "ignore_regex", "Ignoring regex", "Regular expression for lines to be ignored in the threat list.", required_on_create=False, required_on_edit=False), Field("skip_header_lines", "Skip header lines", "Number of header lines to skip, if any. [Defaults to {0}]". format(self.DEFAULT_SKIP_HEADER_LINES), required_on_create=False, required_on_edit=False), # General processing options - should only be set in default stanza. IntegerField( "initial_delay", "Initial delay", """Initial delay in seconds before the modular input begins executing, IF not being executed on a cron schedule. Used to alleviate startup load. [Defaults to {0}]""" .format(self.DEFAULT_INITIAL_DELAY), required_on_create=False, required_on_edit=False), Field("master_host", "Master host", "The master host for this download.", required_on_create=False, required_on_edit=False), ] self._app = 'SA-ThreatIntelligence' self._owner = 'nobody' self._name = 'Threatlist' self._logger = setup_logger(name='threatlist', level=logging.INFO) super(ThreatlistModularInput, self).__init__(scheme_args, args)
import collections import itertools import logging import sys import operator import json import httplib from splunk.persistconn.application import PersistentServerConnectionApplication from splunk.clilib.bundle_paths import make_splunkhome_path import splunk.search sys.path.append(make_splunkhome_path(["etc", "apps", "SA-Utils", "lib"])) from SolnCommon.log import setup_logger, SHORT_FORMAT logger = setup_logger('identitymapper', format=SHORT_FORMAT, level=logging.INFO) if sys.platform == "win32": import os, msvcrt msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY) msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY) class IdentityMapperErrors(object): '''Enum for error strings.''' ERR_INVALID_ARG = 'Invalid argument provided' ERR_INVALID_ARGC = 'Invalid argument count provided' ERR_INVALID_CONSTRAINT_METHOD = 'Invalid constraint method requested' ERR_INVALID_CONSTRAINT_FIELD = 'Invalid constraint field requested' class IdentityMapper(PersistentServerConnectionApplication):
from splunk.appserver.mrsparkle.lib.decorators import expose_page from splunk.appserver.mrsparkle.lib.routes import route from splunk.rest import simpleRequest from splunk.clilib.bundle_paths import make_splunkhome_path sys.path.append(make_splunkhome_path(['etc', 'apps', 'SA-Utils', 'lib'])) from SolnCommon.log import setup_logger, SHORT_FORMAT # Import the correlation search helper class sys.path.append( make_splunkhome_path(["etc", "apps", "SA-ThreatIntelligence", "bin"])) from correlation_search import CorrelationSearch from custom_search_builder.base import CustomSearchBuilderBase from shortcuts import Severity logger = setup_logger('correlation_search_controller', level=logging.DEBUG, format=SHORT_FORMAT) class CorrelationSearchBuilder(controllers.BaseController): '''Correlation search builder Controller''' @route('/:ping=ping') @expose_page(must_login=True, methods=['GET', 'POST']) def ping(self, **kwargs): return self.render_json({'is_available': '1'}, set_mime='text/plain') @route('/:save=save') @expose_page(must_login=True, methods=['POST']) def save(self, **kwargs): """ This handler is currently invoked by the "Save correlation search" button that is included in predictive analytics. You should be using the 'update_or_create_search' for new functionality.
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY) import splunk import splunk.admin import splunk.rest import splunk.util from splunk import ResourceNotFound from splunk.clilib.bundle_paths import make_splunkhome_path from splunk.appserver.mrsparkle.lib import i18n sys.path.append(make_splunkhome_path(["etc", "apps", "SA-Utils", "lib"])) from SolnCommon.log import setup_logger, SHORT_FORMAT logger = setup_logger('identityLookup_rest_handler', level=logging.INFO, format=SHORT_FORMAT) class IdentityLookup(object): # See definitions for convention keys and values in identityLookup.conf.spec. CONVENTION_KEY_RX = re.compile(r'^(convention\.(\d+))$') REQUIRED_PARAMS = [ 'exact', 'email', 'email_short', 'convention', 'case_sensitive' ] OPTIONAL_PARAMS = ['convention.*'] LEGACY_PARAMS = ['match_order'] VALID_PARAMS = REQUIRED_PARAMS + OPTIONAL_PARAMS + LEGACY_PARAMS
sys.path.append(make_splunkhome_path(['etc', 'apps', 'SA-Utils', 'lib'])) from SolnCommon.searchutils import parse_search_string from SolnCommon.log import setup_logger, SHORT_FORMAT # Ensure that shortcuts can be imported. sys.path.append( make_splunkhome_path(["etc", "apps", "SA-ThreatIntelligence", "bin"])) from shortcuts import Duration from shortcuts import Severity from shortcuts import NotableOwner # Import the custom search builder from custom_search_builder.base import CustomSearchBuilderBase from custom_search_builder.make_correlation_search import makeCorrelationSearch logger = setup_logger('correlationsearches_base_class', level=logging.DEBUG, format=SHORT_FORMAT) def error(key): ''' Returns an error message for a given field. ''' # Dictionary of error messages error_descriptions = { 'alert.suppress.fields': "One or more fields must be selected to group by.", 'alert.suppress.period': "Aggregation window duration must be a positive integer." }