def add_logger_splunkhandler(logger=set_default_logger(), **kwargs):
"""
Handler for writing logs to Splunk index.
https://github.com/vavarachen/splunk_hec_handler
:param logger: logging instance
:param kwargs: Splunk configuration options
:return: logger with Splunk Handler attached
"""
try:
from splunk_hec_handler import SplunkHecHandler
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
except Exception as err:
logger.warning("Failed to add Splunk log handler. Error: %s" % err)
return logger
else:
try:
host = kwargs.pop('host')
token = kwargs.pop('token')
level = kwargs.pop('level') if 'level' in kwargs.keys() else 'INFO'
sh = SplunkHecHandler(host, token, **kwargs)
except Exception as err:
logger.warning("Failed to add Splunk log handler. Error: %s" %
err)
raise err
else:
sh.setLevel(level)
logger.addHandler(sh)
return logger
def add_splunk_handler(args):
"""
Add remote Splunk HEC logging handler to logger
:param args: argparse Namespace containing values to configure Splunk handler. Host and Token required.
:return: None. Adds splunk log handler to logger.
"""
if not args.splunk:
return
try:
# Third-Party Libraries
from splunk_hec_handler import SplunkHecHandler
except ModuleNotFoundError as err:
logger.warning(
"Filed to import 'splunk_hec_handler' python module. Try 'pip install splunk_hec_handler'"
)
except Exception as err:
logger.warning(
"Error encountered adding Splunk logging handler. Error: %s" %
err)
else:
if not args.verify:
try:
# Third-Party Libraries
import urllib3
urllib3.disable_warnings(
urllib3.exceptions.InsecureRequestWarning)
except ModuleNotFoundError as err:
logger.debug("Failed to suppress SSL warnings")
logger.debug(
"Configuring Splunk handler: host: %s, port: %d, proto: %s, ssl_verify: %s, token: %s, source: %s, sourcetype: %s"
% (
args.host,
args.port,
args.proto,
args.verify,
args.token,
args.source,
args.sourcetype,
))
splunk_handler = SplunkHecHandler(
args.host,
args.token,
index=args.index,
port=args.port,
proto=args.proto,
ssl_verify=args.verify,
source=args.source,
sourcetype=args.sourcetype,
)
splunk_handler.setLevel(logging.getLevelName(args.loglevel))
logger.addHandler(splunk_handler)
def add_logger_splunkhandler(
logger=set_default_logger(), log_filter=None, **kwargs):
"""
Handler for writing logs to Splunk index.
:param logger: logging instance
:param log_filter: logging Filter object
:param kwargs: Splunk configuration options
:return: logger with Splunk Handler attached
"""
try:
from splunk_hec_handler import SplunkHecHandler
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
except Exception as err:
logger.warning("Failed to add Splunk log handler. Error: %s" % err)
return logger
else:
try:
host = kwargs.pop('host')
token = kwargs.pop('token')
level = kwargs.get('level', 'INFO')
sh = SplunkHecHandler(host, token, **kwargs)
sh.set_name("{}_splunk".format(logger.name))
except Exception as err:
logger.warning("Failed to add Splunk log handler. Error: %s" %
err)
raise err
else:
sh.setLevel(level)
if log_filter is not None:
sh.addFilter(log_filter)
logger.addHandler(sh)
return logger
def send_to_splunk(stats, config):
""" Send the stats to splunk """
# Using this splunk log handler this way is a little odd, but the simplest and most
# robust method I've seen for getting some random object data into splunk via the HTTP Event Collector
# https://pypi.org/project/splunk-hec-handler/
if config['splunk_ssl']:
protocol = 'https'
else:
protocol = 'http'
logger = logging.getLogger('SplunkHecHandlerExample')
logger.setLevel(logging.DEBUG)
# If using self-signed certificate, set ssl_verify to False
# If using http, set proto to http
splunk_handler = SplunkHecHandler(config['splunk_host'],
config['splunk_token'],
port=config['splunk_port'],
proto=protocol,
ssl_verify=config['splunk_verify_ssl'],
source=config['splunk_source'],
sourcetype='_json')
logger.addHandler(splunk_handler)
logger.info(stats)
logging.info('Successfully wrote data to Splunk')
logging.debug('Stats sent to Splunk:')
logging.debug(stats)
def stream_to_splunk(self):
logger = logging.getLogger(self.splunk_source)
logger.setLevel(logging.INFO)
splunk_handler = SplunkHecHandler(self.splunk_host,
self.splunk_token,
port=self.splunk_port,
proto=self.splunk_proto,
ssl_verify=self.splunk_ssl_verify,
source=self.splunk_source)
logger.addHandler(splunk_handler)
consumer = KafkaConsumer(
self.kafka_topic,
bootstrap_servers=self.kafka_brokers,
auto_offset_reset=self.kafka_auto_offset_reset,
enable_auto_commit=self.kafka_enable_autocommit)
for msg in consumer:
if msg.value:
data = {"fields": {}}
try:
msg_value = json.loads(msg.value)
except:
msg_value = msg.value
if type(msg_value) == dict:
for k, v in msg_value.items():
data["fields"].update({str(k): v})
else:
data = msg_value
logger.info(data)
else:
break
def setUp(self):
self.splunk_handler = SplunkHecHandler(
proto=SPLUNK_PROTO,
host=SPLUNK_HOST,
port=SPLUNK_PORT,
token=SPLUNK_TOKEN,
hostname=CLIENT_HOSTNAME,
source=SPLUNK_SOURCE,
sourcetype=SPLUNK_SOURCETYPE,
ssl_verify=SPLUNK_VERIFY,
)
self.splunk_handler.testing = True
domain = 'https://www.rubmaps.com'
client = boto3.client('s3')
bucket = 'ti-merida'
prefix = 'sites/rm/' # Change for the new page folder
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
logger = logging.getLogger('SplunkHECHandler')
logger.setLevel(logging.DEBUG)
MAX_RETRIES = 300
current_state = ''
current_page = 1
splunk_host = "logging.threatinformantlabs.com"
splunk_port = 8088
splunk_token = "cd672cae-5f9c-4d07-854e-d9d650f7b147"
splunk_handler = SplunkHecHandler(splunk_host,
splunk_token,
port=8088,
proto='https',
ssl_verify=False,
source="merida")
logger.addHandler(splunk_handler)
global scraped_places
# -----------------------------------------------------------------------------
def get_html_from_request(url):
'''Returs the html from the url. It uses proxies for every new url'''
proxyDict = {
'http': 'http://*****:*****@69.197.144.122:52027',
'https': 'http://*****:*****@69.197.144.122:52027'
}
headers = {
import logging
from splunk_hec_handler import SplunkHecHandler
logger = logging.getLogger('SplunkHecHandlerExample')
logger.setLevel(logging.DEBUG)
# If using self-signed certificate, set ssl_verify to False
# If using http, set proto to http
splunk_handler = SplunkHecHandler('splunk-dev.sumtotallab.host',
'434634f0-d60a-4592-9959-ba14b3bdccb5',
port=443,
proto='https',
ssl_verify=False,
source="HEC_example",
index="HecTest",
sourcetype="_json")
logger.addHandler(splunk_handler)
logger.debug('This is a debug message')
logger.info('This is an info message')
logger.warning('This is a warning message')
logger.error('This is an error message')
logger.critical('This is a critical message')
a = 5
b = 0
try:
c = a / b
except Exception as e:
logging.exception("Exception occurred")
from splunk_hec_handler import SplunkHecHandler
# Global variables:
# Splunk data for the Python SDK:
HOST = "10.120.163.87"
PORT = 8089
USERNAME = "******"
PASSWORD = "******"
logger = logging.getLogger('SplunkHecHandlerExample')
logger.setLevel(logging.DEBUG)
splunk_handler = SplunkHecHandler('sc1uxpremn81.prod.williamhill.plc',
'd2fb257f-e8e8-455d-8e8e-ff1accacffa0',
port=8088,
proto='https',
ssl_verify=False,
source='jacks-python-script')
logger.addHandler(splunk_handler)
baseDict = {
"log_timestamp": "Timestamp",
"sddc_id": "SDDCID",
"source": "SourceIP",
"priority": "Priority",
"log_type": "LogType",
"hostname": "Hostname",
"event_type": "EventType",
"appname": "AppName",
"ingest_timestamp": "IngestTimestamp",
"id": "ID",
