def add_logger_splunkhandler(logger=set_default_logger(), **kwargs): """ Handler for writing logs to Splunk index. https://github.com/vavarachen/splunk_hec_handler :param logger: logging instance :param kwargs: Splunk configuration options :return: logger with Splunk Handler attached """ try: from splunk_hec_handler import SplunkHecHandler import urllib3 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) except Exception as err: logger.warning("Failed to add Splunk log handler. Error: %s" % err) return logger else: try: host = kwargs.pop('host') token = kwargs.pop('token') level = kwargs.pop('level') if 'level' in kwargs.keys() else 'INFO' sh = SplunkHecHandler(host, token, **kwargs) except Exception as err: logger.warning("Failed to add Splunk log handler. Error: %s" % err) raise err else: sh.setLevel(level) logger.addHandler(sh) return logger
def add_splunk_handler(args): """ Add remote Splunk HEC logging handler to logger :param args: argparse Namespace containing values to configure Splunk handler. Host and Token required. :return: None. Adds splunk log handler to logger. """ if not args.splunk: return try: # Third-Party Libraries from splunk_hec_handler import SplunkHecHandler except ModuleNotFoundError as err: logger.warning( "Filed to import 'splunk_hec_handler' python module. Try 'pip install splunk_hec_handler'" ) except Exception as err: logger.warning( "Error encountered adding Splunk logging handler. Error: %s" % err) else: if not args.verify: try: # Third-Party Libraries import urllib3 urllib3.disable_warnings( urllib3.exceptions.InsecureRequestWarning) except ModuleNotFoundError as err: logger.debug("Failed to suppress SSL warnings") logger.debug( "Configuring Splunk handler: host: %s, port: %d, proto: %s, ssl_verify: %s, token: %s, source: %s, sourcetype: %s" % ( args.host, args.port, args.proto, args.verify, args.token, args.source, args.sourcetype, )) splunk_handler = SplunkHecHandler( args.host, args.token, index=args.index, port=args.port, proto=args.proto, ssl_verify=args.verify, source=args.source, sourcetype=args.sourcetype, ) splunk_handler.setLevel(logging.getLevelName(args.loglevel)) logger.addHandler(splunk_handler)
def add_logger_splunkhandler( logger=set_default_logger(), log_filter=None, **kwargs): """ Handler for writing logs to Splunk index. :param logger: logging instance :param log_filter: logging Filter object :param kwargs: Splunk configuration options :return: logger with Splunk Handler attached """ try: from splunk_hec_handler import SplunkHecHandler import urllib3 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) except Exception as err: logger.warning("Failed to add Splunk log handler. Error: %s" % err) return logger else: try: host = kwargs.pop('host') token = kwargs.pop('token') level = kwargs.get('level', 'INFO') sh = SplunkHecHandler(host, token, **kwargs) sh.set_name("{}_splunk".format(logger.name)) except Exception as err: logger.warning("Failed to add Splunk log handler. Error: %s" % err) raise err else: sh.setLevel(level) if log_filter is not None: sh.addFilter(log_filter) logger.addHandler(sh) return logger
def send_to_splunk(stats, config): """ Send the stats to splunk """ # Using this splunk log handler this way is a little odd, but the simplest and most # robust method I've seen for getting some random object data into splunk via the HTTP Event Collector # https://pypi.org/project/splunk-hec-handler/ if config['splunk_ssl']: protocol = 'https' else: protocol = 'http' logger = logging.getLogger('SplunkHecHandlerExample') logger.setLevel(logging.DEBUG) # If using self-signed certificate, set ssl_verify to False # If using http, set proto to http splunk_handler = SplunkHecHandler(config['splunk_host'], config['splunk_token'], port=config['splunk_port'], proto=protocol, ssl_verify=config['splunk_verify_ssl'], source=config['splunk_source'], sourcetype='_json') logger.addHandler(splunk_handler) logger.info(stats) logging.info('Successfully wrote data to Splunk') logging.debug('Stats sent to Splunk:') logging.debug(stats)
def stream_to_splunk(self): logger = logging.getLogger(self.splunk_source) logger.setLevel(logging.INFO) splunk_handler = SplunkHecHandler(self.splunk_host, self.splunk_token, port=self.splunk_port, proto=self.splunk_proto, ssl_verify=self.splunk_ssl_verify, source=self.splunk_source) logger.addHandler(splunk_handler) consumer = KafkaConsumer( self.kafka_topic, bootstrap_servers=self.kafka_brokers, auto_offset_reset=self.kafka_auto_offset_reset, enable_auto_commit=self.kafka_enable_autocommit) for msg in consumer: if msg.value: data = {"fields": {}} try: msg_value = json.loads(msg.value) except: msg_value = msg.value if type(msg_value) == dict: for k, v in msg_value.items(): data["fields"].update({str(k): v}) else: data = msg_value logger.info(data) else: break
def setUp(self): self.splunk_handler = SplunkHecHandler( proto=SPLUNK_PROTO, host=SPLUNK_HOST, port=SPLUNK_PORT, token=SPLUNK_TOKEN, hostname=CLIENT_HOSTNAME, source=SPLUNK_SOURCE, sourcetype=SPLUNK_SOURCETYPE, ssl_verify=SPLUNK_VERIFY, ) self.splunk_handler.testing = True
domain = 'https://www.rubmaps.com' client = boto3.client('s3') bucket = 'ti-merida' prefix = 'sites/rm/' # Change for the new page folder urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) logger = logging.getLogger('SplunkHECHandler') logger.setLevel(logging.DEBUG) MAX_RETRIES = 300 current_state = '' current_page = 1 splunk_host = "logging.threatinformantlabs.com" splunk_port = 8088 splunk_token = "cd672cae-5f9c-4d07-854e-d9d650f7b147" splunk_handler = SplunkHecHandler(splunk_host, splunk_token, port=8088, proto='https', ssl_verify=False, source="merida") logger.addHandler(splunk_handler) global scraped_places # ----------------------------------------------------------------------------- def get_html_from_request(url): '''Returs the html from the url. It uses proxies for every new url''' proxyDict = { 'http': 'http://*****:*****@69.197.144.122:52027', 'https': 'http://*****:*****@69.197.144.122:52027' } headers = {
import logging from splunk_hec_handler import SplunkHecHandler logger = logging.getLogger('SplunkHecHandlerExample') logger.setLevel(logging.DEBUG) # If using self-signed certificate, set ssl_verify to False # If using http, set proto to http splunk_handler = SplunkHecHandler('splunk-dev.sumtotallab.host', '434634f0-d60a-4592-9959-ba14b3bdccb5', port=443, proto='https', ssl_verify=False, source="HEC_example", index="HecTest", sourcetype="_json") logger.addHandler(splunk_handler) logger.debug('This is a debug message') logger.info('This is an info message') logger.warning('This is a warning message') logger.error('This is an error message') logger.critical('This is a critical message') a = 5 b = 0 try: c = a / b except Exception as e: logging.exception("Exception occurred")
from splunk_hec_handler import SplunkHecHandler # Global variables: # Splunk data for the Python SDK: HOST = "10.120.163.87" PORT = 8089 USERNAME = "******" PASSWORD = "******" logger = logging.getLogger('SplunkHecHandlerExample') logger.setLevel(logging.DEBUG) splunk_handler = SplunkHecHandler('sc1uxpremn81.prod.williamhill.plc', 'd2fb257f-e8e8-455d-8e8e-ff1accacffa0', port=8088, proto='https', ssl_verify=False, source='jacks-python-script') logger.addHandler(splunk_handler) baseDict = { "log_timestamp": "Timestamp", "sddc_id": "SDDCID", "source": "SourceIP", "priority": "Priority", "log_type": "LogType", "hostname": "Hostname", "event_type": "EventType", "appname": "AppName", "ingest_timestamp": "IngestTimestamp", "id": "ID",