def splunkIt(test, result, total_elapsed_time): if app_mode == 'debug': print("Time to Splunk It Yo...\n") logevent = http_event_collector( http_event_collector_key, http_event_collector_host, http_event_port=http_event_collector_port, http_event_server_ssl=http_event_collector_ssl) logevent.popNullFields = True payload = {} payload.update({"index": splunk_index}) payload.update({"sourcetype": splunk_sourcetype}) payload.update({"source": splunk_source}) payload.update({"host": splunk_host}) event = {} event.update({"action": "success"}) event.update({"test": test}) event.update({"total_elapsed_time": total_elapsed_time}) event.update({"test_result": result}) payload.update({"event": event}) logevent.sendEvent(payload) logevent.flushBatch() if app_mode == 'debug': print("It has been Splunked...\n")
def handler(ctx, data: io.BytesIO = None): body = json.loads(data.getvalue()) signer = auth.signers.get_resource_principals_signer() streaming_endpoint = os.environ['streaming_endpoint'] stream_ocid = os.environ['stream_ocid'] body = json.loads(data.getvalue()) offset = body['offset'] num_records = body['num_records'] oci_audit_events_payload = read_from_stream(signer, streaming_endpoint, stream_ocid, offset, num_records) oci_audit_events_JSON = http_event_collector( token=os.environ["splunk_hec_token"], host=os.environ["source_host_name"], input_type="json", http_event_port=os.environ["splunk_hec_port"], http_event_server=os.environ["splunk_url"], ) oci_audit_events_JSON.SSL_verify = False oci_audit_events_JSON.popNullFields = False oci_audit_events_JSON.index = "main" for i in oci_audit_events_payload: payload = {} payload.update({"index": os.environ["splunk_index_name"]}) payload.update({"sourcetype": "_json"}) payload.update({"source": os.environ["splunk_source_name"]}) payload.update({"host": os.environ["source_host_name"]}) payload.update({"event": i}) oci_audit_events_JSON.batchEvent(payload) oci_audit_events_JSON.flushBatch() return response.Response( ctx, response_data=json.dumps({"event": "success"}), headers={"Content-Type": "application/json"}, )
def process_hosts_file(gzfilename, key, logger, host='localhost', batchsize=16384, index='hosts', sourcetype='sonar-host', useesid=False): logger.warning("Loading file {f} at {d}".format(f=gzfilename, d=datetime.now())) hec = http_event_collector(key, host) with gzip.open(gzfilename, 'rb') as resultsfile: m = re.search('.*\/(\d{8})', gzfilename) filedate = m.group(1) filedate_struct = time.strptime(filedate, "%Y%m%d") filedate_epoch = time.mktime(filedate_struct) batchcount = 0 for line in resultsfile: cleanline = line.strip('\n') (host, certhash) = cleanline.split(',', 1) newhost = {} newhost['host'] = host newhost['hash'] = certhash newhost['seen'] = filedate newhost['seen_epoch'] = filedate_epoch if useesid: cert_hash = hashlib.sha1(newhost['host']+newhost['hash']+'sonar') newhost['id'] = cert_hash.hexdigest() newhost = proccess_host(newhost, logger) payload = {} payload.update({"index":index}) payload.update({"host":host}) payload.update({"sourcetype":sourcetype}) payload.update({"source":gzfilename}) payload.update({"event":newhost}) hec.batchEvent(payload) batchcount = batchcount + 1 if batchcount == batchsize: hec.flushBatch() batchcount = 0 if batchcount > 0: hec.flushBatch()
def splunk_data(file_dict): hec_connection_list = [] for hec_profile in hec: for hec_key, hec_values in hec_profile.items(): hec_connection = http_event_collector(http_event_server=hec_values['http_event_server'], http_event_port=hec_values['http_event_port'], token=hec_values['token'], sourcetype=hec_values['sourcetype']) hec_connection_list.append(hec_connection) for key, data in file_dict.items(): payload = {} components = data.get('components') if not components: continue for component in components: new_payload = {} components_flattened = flatten(component,'component') for key, value in data.items(): if not 'components' in key: new_payload[key.lower().strip()] = value.strip().lower() new_payload.update(components_flattened) payload.update({"event": json.dumps(new_payload)}) for hec_connection in hec_connection_list: hec_connection.batchEvent(payload) for hec_connection in hec_connection_list: hec_connection.flushBatch() for hec_connection in hec_connection_list: hec_connection.flushBatch()
def handler(ctx, data: io.BytesIO = None): body = json.loads(data.getvalue()) body = json.loads(data.getvalue()) oci_audit_events_payload = body oci_audit_events_JSON = http_event_collector( token=os.environ["splunk_hec_token"], host=os.environ["source_host_name"], input_type="json", http_event_port=os.environ["splunk_hec_port"], http_event_server=os.environ["splunk_url"], ) oci_audit_events_JSON.SSL_verify = False oci_audit_events_JSON.popNullFields = False oci_audit_events_JSON.index = "main" payload = {} payload.update({"index": os.environ["splunk_index_name"]}) payload.update({"sourcetype": "_json"}) payload.update({"source": os.environ["splunk_source_name"]}) payload.update({"host": os.environ["source_host_name"]}) payload.update({"event": oci_audit_events_payload}) oci_audit_events_JSON.batchEvent(payload) oci_audit_events_JSON.flushBatch() return response.Response( ctx, response_data=json.dumps({"event": "success"}), headers={"Content-Type": "application/json"}, )
def handler(ctx, data: io.BytesIO = None): signer = oci.auth.signers.get_resource_principals_signer() incomingBody = json.loads(data.getvalue()) vcn_flowlog_data = read_from_objectStorage(signer, incomingBody) oci_audit_events_JSON = http_event_collector( token=os.environ["splunk_hec_token"], host=os.environ["source_host_name"], input_type="json", http_event_port=os.environ["splunk_hec_port"], http_event_server=os.environ["splunk_url"], ) oci_audit_events_JSON.SSL_verify = False oci_audit_events_JSON.popNullFields = False oci_audit_events_JSON.index = "main" for i in vcn_flowlog_data: payload = {} payload.update({"index": os.environ["splunk_index_name"]}) payload.update({"sourcetype": "_json"}) payload.update({"source": os.environ["splunk_source_name"]}) payload.update({"host": os.environ["source_host_name"]}) payload.update({"event": i}) oci_audit_events_JSON.batchEvent(payload) oci_audit_events_JSON.flushBatch() return response.Response( ctx, response_data=json.dumps({"event":"success"}), headers={"Content-Type": "application/json"}, )
def process_certs_file(gzfilename, key, logger, host='localhost', batchsize=16384, index='certs', sourcetype='sonar-cert'): logger.warning("Loading file {f} at {d}".format(f=gzfilename, d=datetime.now())) hec = http_event_collector(key, host) with gzip.open(gzfilename, 'rb') as resultsfile: m = re.search('.*\/(\d{8})', gzfilename) filedate = m.group(1) filedate_struct = time.strptime(filedate, "%Y%m%d") filedate_epoch = time.mktime(filedate_struct) batchcount = 0 for line in resultsfile: cleanline = line.strip('\n') (hash_string, cert_b64) = cleanline.split(',', 1) newcert = process_cert(cert_b64, logger) newcert_dict = json.dumps(newcert) payload = {} payload.update({"index":index}) payload.update({"sourcetype":sourcetype}) payload.update({"source":gzfilename}) payload.update({"event":newcert_dict}) hec.batchEvent(payload) batchcount = batchcount + 1 if batchcount == batchsize: hec.flushBatch() batchcount = 0 if batchcount > 0: hec.flushBatch()
def process_hosts_file(file_queue, key, hostlist=['localhost'], index='sonarsslhost', sourcetype='sonarsslhost', batchsize=16384, useesid=False): logger = logging.getLogger('SSLImporter') while True: host = random.choice(hostlist) print host hec = http_event_collector(key, host) gzfilename = file_queue.get() if gzfilename == "DONE": return True logger.warning("Loading file {f} at {d}".format(f=gzfilename, d=datetime.now())) with gzip.open(gzfilename, 'rb') as resultsfile: m = re.search('.*\/(\d{8})', gzfilename) if m: filedate = m.group(1) else: m = re.search('.*\/(\d{4}-\d{2}-\d{2})', gzfilename) filedate = m.group(1) filedate = re.sub('-', '', filedate, 0, 0) filedate_struct = time.strptime(filedate, "%Y%m%d") filedate_epoch = time.mktime(filedate_struct) batchcount = 0 for line in resultsfile: cleanline = line.strip('\n') (host, certhash) = cleanline.split(',', 1) newhost = {} newhost['host'] = host newhost['hash'] = certhash newhost['seen'] = filedate newhost['seen_epoch'] = filedate_epoch if useesid: cert_hash = hashlib.sha1(newhost['host'] + newhost['hash'] + 'sonar') newhost['id'] = cert_hash.hexdigest() newhost = proccess_host(newhost, logger) payload = {} payload.update({"index": index}) payload.update({"host": host}) payload.update({"sourcetype": sourcetype}) payload.update({"source": gzfilename}) payload.update({"event": newhost}) hec.batchEvent(payload) batchcount = batchcount + 1 if batchcount == batchsize: hec.flushBatch() batchcount = 0 if batchcount > 0: hec.flushBatch()
def getTheHECInThere(data): http_event_collector_key = os.environ.get("HEC_KEY") http_event_collector_host = os.environ.get("HEC_HOST") event = http_event_collector(http_event_collector_key, http_event_collector_host) payload = {} payload.update({"index": "owntracks"}) payload.update({"sourcetype": "owntracks"}) payload.update({"source": "mqtt-owntracks"}) payload.update({"host": "tightbeam"}) payload.update({"event": data}) event.sendEvent(payload)
def print_to_hec(event): http_event_collector_key = SPLUNK_HEC_KEY http_event_collector_host = SPLUNK_HEC_HOST testevent = http_event_collector(http_event_collector_key, http_event_collector_host) testevent.popNullFields = True payload = {} payload.update({"index": SPLUNK_HEC_INDEX}) payload.update({"sourcetype": "netgear_cm700"}) payload.update({"source": "netgear_cm700_status"}) payload.update({"host": SPLUNK_EVENT_HOST}) payload.update({"event": event}) testevent.sendEvent(payload) return
def HecLogger(host, token, port, dict_data, source, file_logger, debug): ''' A function to perform logging to Splunk using the HTTP event logger (HEC). ''' event_logger = http_event_collector(token, host) payload = {} payload.update({"sourcetype": "_json"}) payload.update({"source": source}) payload.update({"event": dict_data}) event_logger.sendEvent(payload) event_logger.flushBatch() return True
def configure(self, index: str, nb_ingestors: int): """ Configure the instance of SplunkHelper :param index: Index where to push the files :return: True if successfully configured else False """ # Load the environment variables for .env load_dotenv() self._nb_ingestors = nb_ingestors log.info("Init SplunkHelper") self._sh = SplunkHelper( splunk_url=os.getenv("SPLUNK_URL"), splunk_port=os.getenv("SPLUNK_MPORT"), splunk_ssl_verify=os.getenv("SPLUNK_SSL") == "True", username=os.getenv("SPLUNK_USER"), password=os.getenv("SPLUNK_PASS")) # The SplunkHelper instantiation holds a link_up # flag that indicated whether it could successfully reach # the specified SPlunk instance if self._sh.link_up: # Fetch or create the HEC token from Splunk hect = self._sh.get_or_create_hect() # Create a new index if self._sh.create_index(index=index): # Associate the index to the HEC token so the script can send # the logs to it self._sh.register_index_to_hec(index=index) # Instantiate HEC class and configure self._hec_server = http_event_collector( token=hect, http_event_server=os.getenv("SPLUNK_URL")) self._hec_server.http_event_server_ssl = True self._hec_server.index = index self._hec_server.input_type = "json" self._hec_server.popNullFields = True return True return False
def process_https_file(file_queue, key, hostlist=['localhost'], batchsize=16384, index='sonarhttps', sourcetype='sonar:https'): logger = logging.getLogger("HTTPSImporter") while True: host = random.choice(hostlist) print host hec = http_event_collector(key, host) gzfilename = file_queue.get() if gzfilename == "DONE": return True logger.warning("Loading file {f} at {d}".format(f=gzfilename, d=datetime.now())) with gzip.open(gzfilename, 'rb') as resultsfile: print gzfilename m = re.search('^.*\/(\d{4})\-?(\d{2})\-?(\d{2})', gzfilename) filedate = '{}{}{}'.format(m.group(1), m.group(2), m.group(3)) filedate_struct = time.strptime(filedate, "%Y%m%d") filedate_epoch = time.mktime(filedate_struct) batchcount = 0 for line in resultsfile: new_https_get_dict = {} new_https_get_dict = json.loads(line) new_https_get_dict['data_decoded'] = unicode(base64.b64decode( new_https_get_dict['data']), errors='ignore') payload = {} payload.update({"index": index}) payload.update({"host": host}) payload.update({"sourcetype": sourcetype}) payload.update({"source": gzfilename}) payload.update({"event": new_https_get_dict}) hec.batchEvent(payload) batchcount = batchcount + 1 if batchcount == batchsize: hec.flushBatch() batchcount = 0 if batchcount > 0: hec.flushBatch()
def main(): # Main program block if (len(sys.argv) < 2): print("Usage: sender.py <JSON-data-file>") sys.exit(1) sensor_file = sys.argv[1] # Setup Splunk HEC Connector splunk = http_event_collector(HECKEY, HECHOST) splunk.log.setLevel(logging.ERROR) # Perform a HEC endpoint reachable check hec_reachable = splunk.check_connectivity() if not hec_reachable: print("ERROR: HEC endpoint unreachable.") sys.exit(1) # Read line of file and break apart JSON into event items event = {} try: file = open(sensor_file, "r") data = json.load(file) except: print("ERROR: Unable to open %s" % sensor_file) sys.exit(1) try: for k, v in data.items(): event.update({k: v}) # Build payload with metadata information payload = {} payload.update({"index": INDEX}) payload.update({"sourcetype": SOURCETYPE}) payload.update({"source": SOURCE}) payload.update({"host": HOSTNAME}) payload.update({"event": event}) # Send payload splunk.sendEvent(payload) splunk.flushBatch() except: print("ERROR: Unable to parse file %s" % sensor_file)
def process_certs_file(file_queue, key, hostlist=['localhost'], index='sonarsslcert', sourcetype='sonarsslcert', batchsize=16384): logger = logging.getLogger("SSLImporter") while True: host = random.choice(hostlist) print host hec = http_event_collector(key, host) gzfilename = file_queue.get() if gzfilename == "DONE": return True logger.warning("Loading file {f} at {d}".format(f=gzfilename, d=datetime.now())) with gzip.open(gzfilename, 'rb') as resultsfile: m = re.search('.*\/(\d{8})', gzfilename) if m: filedate = m.group(1) else: m = re.search('.*\/(\d{4}-\d{2}-\d{2})', gzfilename) filedate = m.group(1) filedate = re.sub('-', '', filedate, 0, 0) filedate_struct = time.strptime(filedate, "%Y%m%d") filedate_epoch = time.mktime(filedate_struct) batchcount = 0 for line in resultsfile: cleanline = line.strip('\n') (hash_string, cert_b64) = cleanline.split(',', 1) newcert = process_cert(cert_b64, logger) newcert_dict = json.dumps(newcert) payload = {} payload.update({"index":index}) payload.update({"sourcetype":sourcetype}) payload.update({"source":gzfilename}) payload.update({"event":newcert_dict}) hec.batchEvent(payload) batchcount = batchcount + 1 if batchcount == batchsize: hec.flushBatch() batchcount = 0 if batchcount > 0: hec.flushBatch()
def splunkexporter(host, token, port, dict_data, source, file_logger): ''' A function to perform logging to Splunk using the HTTP event logger (HEC). ''' if not splunk_modules: file_logger.error(" ********* MAJOR ERROR ********** ") file_logger.error( "One or more Splunk Python .are not installed on this system. Splunk export failed, exiting" ) file_logger.error(import_err) sys.exit() event_logger = http_event_collector(token, host) payload = {} payload.update({"sourcetype": "_json"}) payload.update({"source": source}) payload.update({"event": dict_data}) event_logger.sendEvent(payload) event_logger.flushBatch() return True
def send_logs(): for subdir, dirs, files in os.walk(rootdir): if dirs: for directory in dirs: dirpath = rootdir + os.sep + directory moveDirs.append(dirpath) for file in files: filepath = subdir + os.sep + file if filepath.endswith('.gz'): print('Sending {} to Splunk HEC endpoint at {}...'.format(filepath, splunkHost)) with gzip.open(filepath, 'rt') as f: crowdstrike_event = http_event_collector(authToken, splunkHost) payload = {} payload.update({'index': 'crowdstrike_hec'}) payload.update({'sourcetype': 'crowdstrike'}) payload.update({'source': filepath}) payload.update({'host': 'crowdstrike_replicator'}) for line in f: timestamp = extract_timestamp(line) payload.update({'time': timestamp}) payload.update({'event': line}) crowdstrike_event.batchEvent(payload) crowdstrike_event.flushBatch()
def process_event(helper, *args, **kwargs): """ # IMPORTANT # Do not remove the anchor macro:start and macro:end lines. # These lines are used to generate sample code. If they are # removed, the sample code will not be updated when configurations # are updated. [sample_code_macro:start] # The following example gets the alert action parameters and prints them to the log u_splunkserver = helper.get_param("u_splunkserver") helper.log_info("u_splunkserver={}".format(u_splunkserver)) u_splunkserverport = helper.get_param("u_splunkserverport") helper.log_info("u_splunkserverport={}".format(u_splunkserverport)) u_hectoken = helper.get_param("u_hectoken") helper.log_info("u_hectoken={}".format(u_hectoken)) u_senddatatype = helper.get_param("u_senddatatype") helper.log_info("u_senddatatype={}".format(u_senddatatype)) u_destindex = helper.get_param("u_destindex") helper.log_info("u_destindex={}".format(u_destindex)) u_destsourcetype = helper.get_param("u_destsourcetype") helper.log_info("u_destsourcetype={}".format(u_destsourcetype)) u_host = helper.get_param("u_host") helper.log_info("u_host={}".format(u_host)) u_destsource = helper.get_param("u_destsource") helper.log_info("u_destsource={}".format(u_destsource)) # The following example adds two sample events ("hello", "world") # and writes them to Splunk # NOTE: Call helper.writeevents() only once after all events # have been added helper.addevent("hello", sourcetype="sample_sourcetype") helper.addevent("world", sourcetype="sample_sourcetype") helper.writeevents(index="summary", host="localhost", source="localhost") # The following example gets the events that trigger the alert events = helper.get_events() for event in events: helper.log_info("event={}".format(event)) # helper.settings is a dict that includes environment configuration # Example usage: helper.settings["server_uri"] helper.log_info("server_uri={}".format(helper.settings["server_uri"])) [sample_code_macro:end] """ try: from splunk_http_event_collector import http_event_collector import json except ImportError as err_message: helper.log_error("{}".format(err_message)) return 1 helper.log_info("Alert action sendtohec started.") u_splunkserver = helper.get_param("u_splunkserver") u_splunkserverport = helper.get_param("u_splunkserverport") helper.log_info("splunkserver={0}:{1}".format(u_splunkserver, u_splunkserverport)) u_hectoken = helper.get_param("u_hectoken") helper.log_info("u_hectoken={}".format(u_hectoken)) u_senddatatype = helper.get_param("u_senddatatype") helper.log_info("u_senddatatype={}".format(u_senddatatype)) u_destindex = helper.get_param("u_destindex") u_destsourcetype = helper.get_param("u_destsourcetype") u_host = helper.get_param("u_host") u_destsource = helper.get_param("u_destsource") searchResults = helper.get_events() hasSearchResults = False if u_senddatatype == "raw": destCollector = http_event_collector(u_hectoken, u_splunkserver, 'raw', '', u_splunkserverport) else: destCollector = http_event_collector(u_hectoken, u_splunkserver, 'json', '', u_splunkserverport) # Enable popping empty fields destCollector.popNullFields = True if u_senddatatype == "raw": for entry in searchResults: hasSearchResults = True payload = entry.get('_raw') destCollector.batchEvent("{}".format(payload)) if not hasSearchResults: helper.log_info("Empty Search Results") else: destCollector.flushBatch() helper.log_info("Alert action sendtohec completed.") return 0 # handle if send type is JSON for entry in searchResults: hasSearchResults = True payload = {} #handle metadata for event if u_destindex: payload['index'] = u_destindex else: payload['index'] = entry.get('index') if u_destsourcetype: payload['sourcetype'] = u_destsourcetype else: payload['sourcetype'] = entry.get('sourcetype') if u_destsource: payload['source'] = u_destsource else: payload['source'] = entry.get('source') if u_host: payload['host'] = u_host else: payload['host'] = entry.get('host') payload['time'] = entry.get('_time') if 'index' in entry: entry.pop('index') if 'sourcetype' in entry: entry.pop('sourcetype') if 'source' in entry: entry.pop('source') if 'host' in entry: entry.pop('host') # event payload minus hidden fields and reserved splunk fields entry = { k: entry.get(k) for k, v in entry.items() if k.startswith('_') } # clean off Splunk specific fields from payload entry.pop('punct') entry.pop('splunk_server') payload['event'] = entry destCollector.batchEvent(payload) if not hasSearchResults: helper.log_info("Empty Search Results") else: destCollector.flushBatch() helper.log_info("Alert action sendtohec completed.") return 0
import json def commitCrime(): # list of sample values suspects = ['Miss Scarett','Professor Plum','Miss Peacock','Mr. Green','Colonel Mustard','Mrs. White'] weapons = ['candlestick','knife','lead pipe','revolver','rope','wrench'] rooms = ['kitchen','ballroom','conservatory','dining room','cellar','billiard room','library','lounge','hall','study'] return {"killer":random.choice(suspects), "weapon":random.choice(weapons), "location":random.choice(rooms), "victim":"Mr Boddy"} # Create event collector object, default SSL and HTTP Event Collector Port http_event_collector_key = "B02336E2-EEC2-48FF-9FA8-267B553A0C6B" http_event_collector_host = "localhost" testevent = http_event_collector(http_event_collector_key, http_event_collector_host) # Start event payload and add the metadata information payload = {} payload.update({"index":"temp"}) payload.update({"sourcetype":"crime"}) payload.update({"source":"witness"}) payload.update({"host":"mansion"}) # Report 5 Crimes for i in range(5): event = commitCrime() event.update({"action":"success"}) event.update({"crime_type":"single"}) event.update({"crime_number":i}) payload.update({"event":event})
import requests import datetime from splunk_http_event_collector import http_event_collector key = "7f75b2f2-2b0c-41c9-b9ac-beeec40f7410" host = "localhost" testevent = http_event_collector(key, host) testevent.popNullFields = True payload = {} payload.update({"index": "main"}) payload.update({"sourcetype": "Temperature"}) payload.update({"source": "Apartment Temperature Reader"}) payload.update({"host": "Pi"}) payload.update({"event": {"Temp": 22.1}}) result = testevent.sendEvent(payload)
"killer": random.choice(suspects), "weapon": random.choice(weapons), "location": random.choice(rooms), "victim": "Mr Boddy" } # init logging config, this would be job of your main code using this class. logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s', datefmt='%Y-%m-%d %H:%M:%S %z') # Create event collector object, default SSL and HTTP Event Collector Port http_event_collector_key = "4D14F8D9-D788-4E6E-BF2D-D1A46441242E" http_event_collector_host = "localhost" testevent = http_event_collector(http_event_collector_key, http_event_collector_host) # perform a HEC reachable check hec_reachable, hec_conn_msg = testevent.check_connectivity() if not hec_reachable: print("Connection issue: {} (exiting)".format(hec_conn_msg)) sys.exit(1) # Set to pop null fields. Always a good idea testevent.popNullFields = True # set logging to DEBUG for example testevent.log.setLevel(logging.DEBUG) # Start event payload and add the metadata information payload = {} payload.update({"index": "test"})
import os import glob import time import logging from splunk_http_event_collector import http_event_collector splunkHECKey = "7f75b2f2-2b0c-41c9-b9ac-beeec40f7410" splunkHECHost = "192.168.2.11" eventSender = http_event_collector(splunkHECKey, splunkHECHost) os.system('modprobe w1-gpio') os.system('modprobe w1-therm') base_dir = '/sys/bus/w1/devices/' device_folder = glob.glob(base_dir + '28*')[0] device_file = device_folder + '/w1_slave' def read_temp_raw(): f = open(device_file, 'r') lines = f.readlines() f.close() return lines def read_temp(): lines = read_temp_raw() while lines[0].strip()[-3:] != 'YES': time.sleep(0.2) lines = read_temp_raw() equals_pos = lines[1].find('t=') if equals_pos != -1:
def lambda_handler(event, context): http_event_collector_token = os.environ['SPLUNK_HEC_TOKEN'] http_event_collector_host, http_event_collector_port, http_event_collector_path = hostIsolator( os.environ['SPLUNK_HEC_HOST']) eventSenderJSON = http_event_collector( http_event_collector_token, http_event_collector_host, 'json', http_event_port=http_event_collector_port) eventSenderJSON.debug = True session = boto3.session.Session() securityhub = session.client('securityhub', region_name=os.environ['AWS_REGION']) results = getFindings(session, securityhub) fresh_events_after_this_time = freshEventTimestampGenerator() fresh_events = True first_call = True sent_count = 0 while ((first_call or 'NextToken' in results) and fresh_events): # Loop through all findings (20 by default) returned by Security Hub API call # If finding has the string "SENT TO SPLUNK" in the finding note, the event is not sent but # loop will continue. # Fresh events will be sent to Splunk over HTTP Event Collector (HEC), "SENT TO SPLUNK" will # be prefixed to the finding's note. # Break out of the loop when we have looked back across the last hour of events (based on the # finding's LastObservedAt timestamp) first_call = False for finding in results['Findings']: finding_timestamp = findingTimestampGenerator( finding['LastObservedAt']) already_sent = False existing_note = '' principal = 'SplunkSecurityHubLambda' if 'Note' in finding: if 'SENT TO SPLUNK:' in finding['Note']['Text']: already_sent = True else: existing_note = finding['Note']['Text'] principal = finding['Note']['UpdatedBy'] if (finding_timestamp > fresh_events_after_this_time and not already_sent): payload = {} payload.update({'sourcetype': 'aws:securityhub'}) payload.update({'event': json.dumps(finding)}) filters = { 'Id': [{ 'Comparison': 'EQUALS', 'Value': finding['Id'] }], 'LastObservedAt': [{ 'Start': finding['LastObservedAt'], 'End': finding['LastObservedAt'] }], } eventSenderJSON.sendEvent(payload) if not eventSenderJSON.failedToSend: print('Event successfully sent to Splunk') eventSenderJSON.failedToSend = False updateFindingNote(session, securityhub, existing_note, principal, filters) sent_count += 1 else: print('Event NOT successfully sent to Splunk') else: fresh_events = False break if (fresh_events): results = getFindingsWithToken(session, securityhub, results['NextToken']) print '%s findings sent to Splunk: %s' % (sent_count, eventSenderJSON.server_uri)
from subprocess import PIPE, Popen, check_output from PIL import Image, ImageDraw, ImageFont from fonts.ttf import RobotoMedium as UserFont from enviroplus import gas from splunk_http_event_collector import http_event_collector import socket import os try: from smbus2 import SMBus except ImportError: from smbus import SMBus # setup send to splunk try: events = http_event_collector(os.getenv("HEC_TOKEN"), "192.168.99.60") except Exception as e: print(f"missing env key e={e}") sys.exit(-1) hec_payload = {} bus = SMBus(1) # Create BME280 instance bme280 = BME280(i2c_dev=bus) # Create LCD instance disp = ST7735.ST7735(port=0, cs=1, dc=9, backlight=12,
try: from splunk_http_event_collector import http_event_collector #type:ignore except ImportError as error_message: sys.exit(f"Failed to import splunk_http_event_collector: {error_message}") from . import API if __name__ == '__main__': if not os.getenv("PARENT_HOST"): sys.exit("PARENT_HOST environment variable is not set, bailing.") api = API() hec = http_event_collector( token=os.getenv('HECTOKEN'), http_event_server=os.getenv('HECHOST'), http_event_port=os.getenv('HECPORT', "443"), http_event_server_ssl=True, ) hec.index = os.getenv("HECINDEX", "torrent") hec.log.setLevel(logging.DEBUG) # pylint: disable=invalid-name queue_counter = 0 hostname = gethostname() for torrent in api.get_torrents(): # add this so we can troubleshoot later torrent["CONTAINER_ID"] = hostname payload = { "sourcetype": os.getenv('HECSOURCETYPE', "torrent:info"),
def __init__(self): super().__init__() self.HEC_sender = http_event_collector(self.http_event_collector_key, self.http_event_collector_host)
def Output(hashes_list, stats): if debug: print("\n\nMODULE OUTPUT") if debug: print("FUNCTION output.Output") details = "" vt_malware_threshold = int(config.get('config', 'vt_malware_threshold')) #Setup output options #Local local = config.get('config', 'local') if local == 'yes': history = open('history.csv', 'a') size = os.path.getsize('history.csv') #If new file, output header if size == 0: output = "timestamp,environment,computer,file,path,size,md5,sha256,Source,wf_malware,wf_new,wf_upload,vt_positive,vt_total,vt_link,vt_new,vt_upload\n" history.write(output) if debug: print(" Will output to local file") #Splunk splunk = config.get('config', 'splunk') if splunk == 'yes': http_event_collector_key = config.get('config', 'splunk_key') http_event_collector_host = config.get('config', 'splunk_host') event = http_event_collector(http_event_collector_key, http_event_collector_host) payload = {} payload.update({"index": config.get('config', 'splunk_index')}) payload.update( {"sourcetype": config.get('config', 'splunk_sourcetype')}) payload.update({"source": config.get('config', 'splunk_source')}) if debug: print(" Will output to Splunk") #Slack slack = config.get('config', 'slack') if slack == 'yes': token = config.get('config', 'slack_token') username = config.get('config', 'slack_username') channel = config.get('config', 'slack_channel') if debug: print(" Will output to Slack") #Email email = config.get('config', 'email') email_attachments = [] if debug and email == 'yes': print(" Will output to Email") now = datetime.datetime.now() timestamp = str(now.year) + '-' + '{:02d}'.format( now.month) + '-' + '{:02d}'.format(now.day) + ' ' + str( now.hour) + ':' + str(now.minute) #Output results to selected options for hash in hashes_list: computer = hash[0] file = hash[1] path = hash[2] size = hash[3] md5 = hash[4] sha256 = hash[5] source = hash[6] wf_malware = str(hash[7]) wf_new = str(hash[8]) wf_upload = str(hash[9]) vt_positive = str(hash[10]) vt_total = str(hash[11]) vt_link = hash[12] vt_new = str(hash[13]) vt_upload = str(hash[14]) #Count malware hashes if wf_malware == 'yes' or int(vt_positive) >= vt_malware_threshold: stats['malware'] += 1 #Output to CSV if local == 'yes': output = timestamp + ',' + environment + ',' + computer + ',' + file + ',' + path + ',' + size + ',' + md5 + ',' + sha256 + ',' + source + ',' + wf_malware + ',' + wf_new + ',' + wf_upload + ',' + vt_positive + ',' + vt_total + ',' + vt_link + ',' + vt_new + ',' + vt_upload + '\n' history.write(output) #Output to Splunk if splunk == 'yes': event_data = { 'timestamp': timestamp, 'environment': environment, 'type': 'detail', 'hash_source': 'index', 'computer': computer, 'file': file, 'path': path, 'size': size, 'md5': md5, 'sha256': sha256, 'source': source, 'wf_malware': wf_malware, 'wf_new': wf_new, 'wf_upload': wf_upload, 'vt_positive': vt_positive, 'vt_total': vt_total, 'vt_link': vt_link, 'vt_new': vt_new, 'vt_upload': vt_upload } payload.update({'event': event_data}) event.batchEvent(payload) #Output to Slack only if malware if slack == 'yes' and (wf_malware == 'yes' or int(vt_positive) >= vt_malware_threshold): text = ("MALWARE FOUND" + \ "\nTimestamp: " + timestamp + \ "\nComputer: " + computer + \ "\nFile: " + file + \ "\nPath: " + path + \ "\nMD5: " + md5 + \ "\nSHA256: " + sha256 + \ "\nWildFire Malware: " + wf_malware + \ "\nVirusTotal: " + str(vt_positive) + '/' + str(vt_total) + \ "\nVirusTotal Link: " + vt_link) message = urllib2.quote(text) #results = 'emptyStatus' try: url = "https://slack.com/api/chat.postMessage?token=" + token + "&channel=" + channel + "&text=" + message + "&username="******"&pretty=1" req = urllib2.Request(url) response = urllib2.urlopen(req) results = response.read() except (urllib2.HTTPError) as e: results = e if wf_malware == 'yes' or int(vt_positive) >= vt_malware_threshold: details += "\nComputer: " + computer details += "\nFile: " + file details += "\nPath: " + path details += "\nMD5: " + md5 details += "\nSHA256: " + sha256 details += "\nWildFire malware: " + wf_malware details += "\nVirusTotal: " + str(vt_positive) + '/' + str( vt_total) details += "\nVirusTotal link: " + vt_link + '\n' email_attachments.append("reports\\" + md5 + ".pdf") #Output Final statistics and details computers_total = str(stats['computers_total']) computers_hashes = str(stats['computers_hashes']) total = str(stats['total']) excluded = str(stats['excluded']) unique = str(stats['unique']) wf_cache = str(stats['wf_cache']) wf_new = str(stats['wf_new']) wf_uploaded = str(stats['wf_uploaded']) vt_cache = str(stats['vt_cache']) vt_new = str(stats['vt_new']) vt_uploaded = str(stats['vt_uploaded']) malware = str(stats['malware']) email_content = "\nTanFire Statistics" + \ "\nComputers (total/with hashes): " + computers_total + '/' + computers_hashes + \ "\nNew Tanium Index hashes: " + total + \ "\nExcluded: " + excluded + \ "\nUnique: " + unique + \ "\nWildFire cache: " + wf_cache + \ "\nWildFire direct: " + wf_new + \ "\nWildFire uploaded: " + wf_uploaded + \ "\nVirusTotal cache: " + vt_cache + \ "\nVirusTotal direct: " + vt_new + \ "\nMalware: " + malware + \ "\n\nDetails on malware hashes: " + details if debug: print(email_content) if local == 'yes': stats_file = open('stats.csv', 'a') size = os.path.getsize('stats.csv') #If new file, output header if size == 0: output = "timestamp,type,computers_total,computers_hashes,total,excluded,unique,wf_cache,wf_new,wf_ploaded,vt_cache,vt_new,vt_uploaded,malware\n" stats_file.write(output) output = timestamp + ',index,' + computers_total + ',' + computers_hashes + ',' + total + ',' + excluded + ',' + unique + ',' + wf_cache + ',' + wf_new + ',' + wf_uploaded + ',' + vt_cache + ',' + vt_new + ',' + vt_uploaded + ',' + malware + '\n' stats_file.write(output) stats_file.close() if splunk == 'yes': event_data = { 'timestamp': timestamp, 'type': 'summary', 'environment': environment, 'hash_source': 'index', 'computers_total': computers_total, 'computers_hashes': computers_hashes, 'total': total, 'excluded': excluded, 'unique': unique, 'wf_cache': wf_cache, 'wf_new': wf_new, 'wf_uploaded': wf_uploaded, 'vt_cache': vt_cache, 'vt_new': vt_new, 'vt_uploaded': vt_uploaded, 'malware': malware } payload.update({'event': event_data}) event.batchEvent(payload) #Output to Email if email == 'yes': Email(timestamp, email_content, email_attachments) #Output cleanup if local == 'yes': history.close() if splunk == 'yes': event.flushBatch()
return { "killer": random.choice(suspects), "weapon": random.choice(weapons), "location": random.choice(rooms), "victim": "Mr Boddy" } # Create event collector object, default SSL and HTTP Event Collector Port http_event_collector_key = "DB84F19F-B2F1-4B89-BB38-643DFB641B34" http_event_collector_host = "45.55.161.5" testevent = http_event_collector(http_event_collector_key, http_event_collector_host, input_type='json', host="", http_event_port='8088', http_event_server_ssl=True, http_event_collector_debug=True) # Start event payload and add the metadata information payload = {} payload.update({"index": "http-test"}) payload.update({"sourcetype": "crime"}) payload.update({"source": "witness"}) payload.update({"host": "mansion"}) # Report 5 Crimes for i in range(5): event = commitCrime() event.update({"action": "success"})