def splunkIt(test, result, total_elapsed_time):
if app_mode == 'debug':
print("Time to Splunk It Yo...\n")
logevent = http_event_collector(
http_event_collector_key,
http_event_collector_host,
http_event_port=http_event_collector_port,
http_event_server_ssl=http_event_collector_ssl)
logevent.popNullFields = True
payload = {}
payload.update({"index": splunk_index})
payload.update({"sourcetype": splunk_sourcetype})
payload.update({"source": splunk_source})
payload.update({"host": splunk_host})
event = {}
event.update({"action": "success"})
event.update({"test": test})
event.update({"total_elapsed_time": total_elapsed_time})
event.update({"test_result": result})
payload.update({"event": event})
logevent.sendEvent(payload)
logevent.flushBatch()
if app_mode == 'debug':
print("It has been Splunked...\n")
def handler(ctx, data: io.BytesIO = None):
body = json.loads(data.getvalue())
signer = auth.signers.get_resource_principals_signer()
streaming_endpoint = os.environ['streaming_endpoint']
stream_ocid = os.environ['stream_ocid']
body = json.loads(data.getvalue())
offset = body['offset']
num_records = body['num_records']
oci_audit_events_payload = read_from_stream(signer, streaming_endpoint,
stream_ocid, offset,
num_records)
oci_audit_events_JSON = http_event_collector(
token=os.environ["splunk_hec_token"],
host=os.environ["source_host_name"],
input_type="json",
http_event_port=os.environ["splunk_hec_port"],
http_event_server=os.environ["splunk_url"],
)
oci_audit_events_JSON.SSL_verify = False
oci_audit_events_JSON.popNullFields = False
oci_audit_events_JSON.index = "main"
for i in oci_audit_events_payload:
payload = {}
payload.update({"index": os.environ["splunk_index_name"]})
payload.update({"sourcetype": "_json"})
payload.update({"source": os.environ["splunk_source_name"]})
payload.update({"host": os.environ["source_host_name"]})
payload.update({"event": i})
oci_audit_events_JSON.batchEvent(payload)
oci_audit_events_JSON.flushBatch()
return response.Response(
ctx,
response_data=json.dumps({"event": "success"}),
headers={"Content-Type": "application/json"},
)
def process_hosts_file(gzfilename, key, logger, host='localhost', batchsize=16384, index='hosts', sourcetype='sonar-host', useesid=False):
logger.warning("Loading file {f} at {d}".format(f=gzfilename, d=datetime.now()))
hec = http_event_collector(key, host)
with gzip.open(gzfilename, 'rb') as resultsfile:
m = re.search('.*\/(\d{8})', gzfilename)
filedate = m.group(1)
filedate_struct = time.strptime(filedate, "%Y%m%d")
filedate_epoch = time.mktime(filedate_struct)
batchcount = 0
for line in resultsfile:
cleanline = line.strip('\n')
(host, certhash) = cleanline.split(',', 1)
newhost = {}
newhost['host'] = host
newhost['hash'] = certhash
newhost['seen'] = filedate
newhost['seen_epoch'] = filedate_epoch
if useesid:
cert_hash = hashlib.sha1(newhost['host']+newhost['hash']+'sonar')
newhost['id'] = cert_hash.hexdigest()
newhost = proccess_host(newhost, logger)
payload = {}
payload.update({"index":index})
payload.update({"host":host})
payload.update({"sourcetype":sourcetype})
payload.update({"source":gzfilename})
payload.update({"event":newhost})
hec.batchEvent(payload)
batchcount = batchcount + 1
if batchcount == batchsize:
hec.flushBatch()
batchcount = 0
if batchcount > 0:
hec.flushBatch()
def splunk_data(file_dict):
hec_connection_list = []
for hec_profile in hec:
for hec_key, hec_values in hec_profile.items():
hec_connection = http_event_collector(http_event_server=hec_values['http_event_server'],
http_event_port=hec_values['http_event_port'],
token=hec_values['token'],
sourcetype=hec_values['sourcetype'])
hec_connection_list.append(hec_connection)
for key, data in file_dict.items():
payload = {}
components = data.get('components')
if not components:
continue
for component in components:
new_payload = {}
components_flattened = flatten(component,'component')
for key, value in data.items():
if not 'components' in key:
new_payload[key.lower().strip()] = value.strip().lower()
new_payload.update(components_flattened)
payload.update({"event": json.dumps(new_payload)})
for hec_connection in hec_connection_list:
hec_connection.batchEvent(payload)
for hec_connection in hec_connection_list:
hec_connection.flushBatch()
for hec_connection in hec_connection_list:
hec_connection.flushBatch()
def handler(ctx, data: io.BytesIO = None):
body = json.loads(data.getvalue())
body = json.loads(data.getvalue())
oci_audit_events_payload = body
oci_audit_events_JSON = http_event_collector(
token=os.environ["splunk_hec_token"],
host=os.environ["source_host_name"],
input_type="json",
http_event_port=os.environ["splunk_hec_port"],
http_event_server=os.environ["splunk_url"],
)
oci_audit_events_JSON.SSL_verify = False
oci_audit_events_JSON.popNullFields = False
oci_audit_events_JSON.index = "main"
payload = {}
payload.update({"index": os.environ["splunk_index_name"]})
payload.update({"sourcetype": "_json"})
payload.update({"source": os.environ["splunk_source_name"]})
payload.update({"host": os.environ["source_host_name"]})
payload.update({"event": oci_audit_events_payload})
oci_audit_events_JSON.batchEvent(payload)
oci_audit_events_JSON.flushBatch()
return response.Response(
ctx,
response_data=json.dumps({"event": "success"}),
headers={"Content-Type": "application/json"},
)
def handler(ctx, data: io.BytesIO = None):
signer = oci.auth.signers.get_resource_principals_signer()
incomingBody = json.loads(data.getvalue())
vcn_flowlog_data = read_from_objectStorage(signer, incomingBody)
oci_audit_events_JSON = http_event_collector(
token=os.environ["splunk_hec_token"],
host=os.environ["source_host_name"],
input_type="json",
http_event_port=os.environ["splunk_hec_port"],
http_event_server=os.environ["splunk_url"],
)
oci_audit_events_JSON.SSL_verify = False
oci_audit_events_JSON.popNullFields = False
oci_audit_events_JSON.index = "main"
for i in vcn_flowlog_data:
payload = {}
payload.update({"index": os.environ["splunk_index_name"]})
payload.update({"sourcetype": "_json"})
payload.update({"source": os.environ["splunk_source_name"]})
payload.update({"host": os.environ["source_host_name"]})
payload.update({"event": i})
oci_audit_events_JSON.batchEvent(payload)
oci_audit_events_JSON.flushBatch()
return response.Response(
ctx,
response_data=json.dumps({"event":"success"}),
headers={"Content-Type": "application/json"},
)
def process_certs_file(gzfilename, key, logger, host='localhost', batchsize=16384, index='certs', sourcetype='sonar-cert'):
logger.warning("Loading file {f} at {d}".format(f=gzfilename, d=datetime.now()))
hec = http_event_collector(key, host)
with gzip.open(gzfilename, 'rb') as resultsfile:
m = re.search('.*\/(\d{8})', gzfilename)
filedate = m.group(1)
filedate_struct = time.strptime(filedate, "%Y%m%d")
filedate_epoch = time.mktime(filedate_struct)
batchcount = 0
for line in resultsfile:
cleanline = line.strip('\n')
(hash_string, cert_b64) = cleanline.split(',', 1)
newcert = process_cert(cert_b64, logger)
newcert_dict = json.dumps(newcert)
payload = {}
payload.update({"index":index})
payload.update({"sourcetype":sourcetype})
payload.update({"source":gzfilename})
payload.update({"event":newcert_dict})
hec.batchEvent(payload)
batchcount = batchcount + 1
if batchcount == batchsize:
hec.flushBatch()
batchcount = 0
if batchcount > 0:
hec.flushBatch()
def process_hosts_file(file_queue,
key,
hostlist=['localhost'],
index='sonarsslhost',
sourcetype='sonarsslhost',
batchsize=16384,
useesid=False):
logger = logging.getLogger('SSLImporter')
while True:
host = random.choice(hostlist)
print host
hec = http_event_collector(key, host)
gzfilename = file_queue.get()
if gzfilename == "DONE":
return True
logger.warning("Loading file {f} at {d}".format(f=gzfilename,
d=datetime.now()))
with gzip.open(gzfilename, 'rb') as resultsfile:
m = re.search('.*\/(\d{8})', gzfilename)
if m:
filedate = m.group(1)
else:
m = re.search('.*\/(\d{4}-\d{2}-\d{2})', gzfilename)
filedate = m.group(1)
filedate = re.sub('-', '', filedate, 0, 0)
filedate_struct = time.strptime(filedate, "%Y%m%d")
filedate_epoch = time.mktime(filedate_struct)
batchcount = 0
for line in resultsfile:
cleanline = line.strip('\n')
(host, certhash) = cleanline.split(',', 1)
newhost = {}
newhost['host'] = host
newhost['hash'] = certhash
newhost['seen'] = filedate
newhost['seen_epoch'] = filedate_epoch
if useesid:
cert_hash = hashlib.sha1(newhost['host'] +
newhost['hash'] + 'sonar')
newhost['id'] = cert_hash.hexdigest()
newhost = proccess_host(newhost, logger)
payload = {}
payload.update({"index": index})
payload.update({"host": host})
payload.update({"sourcetype": sourcetype})
payload.update({"source": gzfilename})
payload.update({"event": newhost})
hec.batchEvent(payload)
batchcount = batchcount + 1
if batchcount == batchsize:
hec.flushBatch()
batchcount = 0
if batchcount > 0:
hec.flushBatch()
def getTheHECInThere(data):
http_event_collector_key = os.environ.get("HEC_KEY")
http_event_collector_host = os.environ.get("HEC_HOST")
event = http_event_collector(http_event_collector_key,
http_event_collector_host)
payload = {}
payload.update({"index": "owntracks"})
payload.update({"sourcetype": "owntracks"})
payload.update({"source": "mqtt-owntracks"})
payload.update({"host": "tightbeam"})
payload.update({"event": data})
event.sendEvent(payload)
def print_to_hec(event):
http_event_collector_key = SPLUNK_HEC_KEY
http_event_collector_host = SPLUNK_HEC_HOST
testevent = http_event_collector(http_event_collector_key,
http_event_collector_host)
testevent.popNullFields = True
payload = {}
payload.update({"index": SPLUNK_HEC_INDEX})
payload.update({"sourcetype": "netgear_cm700"})
payload.update({"source": "netgear_cm700_status"})
payload.update({"host": SPLUNK_EVENT_HOST})
payload.update({"event": event})
testevent.sendEvent(payload)
return
def HecLogger(host, token, port, dict_data, source, file_logger, debug):
'''
A function to perform logging to Splunk using the HTTP event logger (HEC).
'''
event_logger = http_event_collector(token, host)
payload = {}
payload.update({"sourcetype": "_json"})
payload.update({"source": source})
payload.update({"event": dict_data})
event_logger.sendEvent(payload)
event_logger.flushBatch()
return True
def configure(self, index: str, nb_ingestors: int):
"""
Configure the instance of SplunkHelper
:param index: Index where to push the files
:return: True if successfully configured else False
"""
# Load the environment variables for .env
load_dotenv()
self._nb_ingestors = nb_ingestors
log.info("Init SplunkHelper")
self._sh = SplunkHelper(
splunk_url=os.getenv("SPLUNK_URL"),
splunk_port=os.getenv("SPLUNK_MPORT"),
splunk_ssl_verify=os.getenv("SPLUNK_SSL") == "True",
username=os.getenv("SPLUNK_USER"),
password=os.getenv("SPLUNK_PASS"))
# The SplunkHelper instantiation holds a link_up
# flag that indicated whether it could successfully reach
# the specified SPlunk instance
if self._sh.link_up:
# Fetch or create the HEC token from Splunk
hect = self._sh.get_or_create_hect()
# Create a new index
if self._sh.create_index(index=index):
# Associate the index to the HEC token so the script can send
# the logs to it
self._sh.register_index_to_hec(index=index)
# Instantiate HEC class and configure
self._hec_server = http_event_collector(
token=hect, http_event_server=os.getenv("SPLUNK_URL"))
self._hec_server.http_event_server_ssl = True
self._hec_server.index = index
self._hec_server.input_type = "json"
self._hec_server.popNullFields = True
return True
return False
def process_https_file(file_queue,
key,
hostlist=['localhost'],
batchsize=16384,
index='sonarhttps',
sourcetype='sonar:https'):
logger = logging.getLogger("HTTPSImporter")
while True:
host = random.choice(hostlist)
print host
hec = http_event_collector(key, host)
gzfilename = file_queue.get()
if gzfilename == "DONE":
return True
logger.warning("Loading file {f} at {d}".format(f=gzfilename,
d=datetime.now()))
with gzip.open(gzfilename, 'rb') as resultsfile:
print gzfilename
m = re.search('^.*\/(\d{4})\-?(\d{2})\-?(\d{2})', gzfilename)
filedate = '{}{}{}'.format(m.group(1), m.group(2), m.group(3))
filedate_struct = time.strptime(filedate, "%Y%m%d")
filedate_epoch = time.mktime(filedate_struct)
batchcount = 0
for line in resultsfile:
new_https_get_dict = {}
new_https_get_dict = json.loads(line)
new_https_get_dict['data_decoded'] = unicode(base64.b64decode(
new_https_get_dict['data']),
errors='ignore')
payload = {}
payload.update({"index": index})
payload.update({"host": host})
payload.update({"sourcetype": sourcetype})
payload.update({"source": gzfilename})
payload.update({"event": new_https_get_dict})
hec.batchEvent(payload)
batchcount = batchcount + 1
if batchcount == batchsize:
hec.flushBatch()
batchcount = 0
if batchcount > 0:
hec.flushBatch()
def main():
# Main program block
if (len(sys.argv) < 2):
print("Usage: sender.py <JSON-data-file>")
sys.exit(1)
sensor_file = sys.argv[1]
# Setup Splunk HEC Connector
splunk = http_event_collector(HECKEY, HECHOST)
splunk.log.setLevel(logging.ERROR)
# Perform a HEC endpoint reachable check
hec_reachable = splunk.check_connectivity()
if not hec_reachable:
print("ERROR: HEC endpoint unreachable.")
sys.exit(1)
# Read line of file and break apart JSON into event items
event = {}
try:
file = open(sensor_file, "r")
data = json.load(file)
except:
print("ERROR: Unable to open %s" % sensor_file)
sys.exit(1)
try:
for k, v in data.items():
event.update({k: v})
# Build payload with metadata information
payload = {}
payload.update({"index": INDEX})
payload.update({"sourcetype": SOURCETYPE})
payload.update({"source": SOURCE})
payload.update({"host": HOSTNAME})
payload.update({"event": event})
# Send payload
splunk.sendEvent(payload)
splunk.flushBatch()
except:
print("ERROR: Unable to parse file %s" % sensor_file)
def process_certs_file(file_queue, key, hostlist=['localhost'], index='sonarsslcert', sourcetype='sonarsslcert', batchsize=16384):
logger = logging.getLogger("SSLImporter")
while True:
host = random.choice(hostlist)
print host
hec = http_event_collector(key, host)
gzfilename = file_queue.get()
if gzfilename == "DONE":
return True
logger.warning("Loading file {f} at {d}".format(f=gzfilename, d=datetime.now()))
with gzip.open(gzfilename, 'rb') as resultsfile:
m = re.search('.*\/(\d{8})', gzfilename)
if m:
filedate = m.group(1)
else:
m = re.search('.*\/(\d{4}-\d{2}-\d{2})', gzfilename)
filedate = m.group(1)
filedate = re.sub('-', '', filedate, 0, 0)
filedate_struct = time.strptime(filedate, "%Y%m%d")
filedate_epoch = time.mktime(filedate_struct)
batchcount = 0
for line in resultsfile:
cleanline = line.strip('\n')
(hash_string, cert_b64) = cleanline.split(',', 1)
newcert = process_cert(cert_b64, logger)
newcert_dict = json.dumps(newcert)
payload = {}
payload.update({"index":index})
payload.update({"sourcetype":sourcetype})
payload.update({"source":gzfilename})
payload.update({"event":newcert_dict})
hec.batchEvent(payload)
batchcount = batchcount + 1
if batchcount == batchsize:
hec.flushBatch()
batchcount = 0
if batchcount > 0:
hec.flushBatch()
def splunkexporter(host, token, port, dict_data, source, file_logger):
'''
A function to perform logging to Splunk using the HTTP event logger (HEC).
'''
if not splunk_modules:
file_logger.error(" ********* MAJOR ERROR ********** ")
file_logger.error(
"One or more Splunk Python .are not installed on this system. Splunk export failed, exiting"
)
file_logger.error(import_err)
sys.exit()
event_logger = http_event_collector(token, host)
payload = {}
payload.update({"sourcetype": "_json"})
payload.update({"source": source})
payload.update({"event": dict_data})
event_logger.sendEvent(payload)
event_logger.flushBatch()
return True
def send_logs():
for subdir, dirs, files in os.walk(rootdir):
if dirs:
for directory in dirs:
dirpath = rootdir + os.sep + directory
moveDirs.append(dirpath)
for file in files:
filepath = subdir + os.sep + file
if filepath.endswith('.gz'):
print('Sending {} to Splunk HEC endpoint at {}...'.format(filepath, splunkHost))
with gzip.open(filepath, 'rt') as f:
crowdstrike_event = http_event_collector(authToken, splunkHost)
payload = {}
payload.update({'index': 'crowdstrike_hec'})
payload.update({'sourcetype': 'crowdstrike'})
payload.update({'source': filepath})
payload.update({'host': 'crowdstrike_replicator'})
for line in f:
timestamp = extract_timestamp(line)
payload.update({'time': timestamp})
payload.update({'event': line})
crowdstrike_event.batchEvent(payload)
crowdstrike_event.flushBatch()
def process_event(helper, *args, **kwargs):
"""
# IMPORTANT
# Do not remove the anchor macro:start and macro:end lines.
# These lines are used to generate sample code. If they are
# removed, the sample code will not be updated when configurations
# are updated.
[sample_code_macro:start]
# The following example gets the alert action parameters and prints them to the log
u_splunkserver = helper.get_param("u_splunkserver")
helper.log_info("u_splunkserver={}".format(u_splunkserver))
u_splunkserverport = helper.get_param("u_splunkserverport")
helper.log_info("u_splunkserverport={}".format(u_splunkserverport))
u_hectoken = helper.get_param("u_hectoken")
helper.log_info("u_hectoken={}".format(u_hectoken))
u_senddatatype = helper.get_param("u_senddatatype")
helper.log_info("u_senddatatype={}".format(u_senddatatype))
u_destindex = helper.get_param("u_destindex")
helper.log_info("u_destindex={}".format(u_destindex))
u_destsourcetype = helper.get_param("u_destsourcetype")
helper.log_info("u_destsourcetype={}".format(u_destsourcetype))
u_host = helper.get_param("u_host")
helper.log_info("u_host={}".format(u_host))
u_destsource = helper.get_param("u_destsource")
helper.log_info("u_destsource={}".format(u_destsource))
# The following example adds two sample events ("hello", "world")
# and writes them to Splunk
# NOTE: Call helper.writeevents() only once after all events
# have been added
helper.addevent("hello", sourcetype="sample_sourcetype")
helper.addevent("world", sourcetype="sample_sourcetype")
helper.writeevents(index="summary", host="localhost", source="localhost")
# The following example gets the events that trigger the alert
events = helper.get_events()
for event in events:
helper.log_info("event={}".format(event))
# helper.settings is a dict that includes environment configuration
# Example usage: helper.settings["server_uri"]
helper.log_info("server_uri={}".format(helper.settings["server_uri"]))
[sample_code_macro:end]
"""
try:
from splunk_http_event_collector import http_event_collector
import json
except ImportError as err_message:
helper.log_error("{}".format(err_message))
return 1
helper.log_info("Alert action sendtohec started.")
u_splunkserver = helper.get_param("u_splunkserver")
u_splunkserverport = helper.get_param("u_splunkserverport")
helper.log_info("splunkserver={0}:{1}".format(u_splunkserver,
u_splunkserverport))
u_hectoken = helper.get_param("u_hectoken")
helper.log_info("u_hectoken={}".format(u_hectoken))
u_senddatatype = helper.get_param("u_senddatatype")
helper.log_info("u_senddatatype={}".format(u_senddatatype))
u_destindex = helper.get_param("u_destindex")
u_destsourcetype = helper.get_param("u_destsourcetype")
u_host = helper.get_param("u_host")
u_destsource = helper.get_param("u_destsource")
searchResults = helper.get_events()
hasSearchResults = False
if u_senddatatype == "raw":
destCollector = http_event_collector(u_hectoken, u_splunkserver, 'raw',
'', u_splunkserverport)
else:
destCollector = http_event_collector(u_hectoken, u_splunkserver,
'json', '', u_splunkserverport)
# Enable popping empty fields
destCollector.popNullFields = True
if u_senddatatype == "raw":
for entry in searchResults:
hasSearchResults = True
payload = entry.get('_raw')
destCollector.batchEvent("{}".format(payload))
if not hasSearchResults:
helper.log_info("Empty Search Results")
else:
destCollector.flushBatch()
helper.log_info("Alert action sendtohec completed.")
return 0
# handle if send type is JSON
for entry in searchResults:
hasSearchResults = True
payload = {}
#handle metadata for event
if u_destindex:
payload['index'] = u_destindex
else:
payload['index'] = entry.get('index')
if u_destsourcetype:
payload['sourcetype'] = u_destsourcetype
else:
payload['sourcetype'] = entry.get('sourcetype')
if u_destsource:
payload['source'] = u_destsource
else:
payload['source'] = entry.get('source')
if u_host:
payload['host'] = u_host
else:
payload['host'] = entry.get('host')
payload['time'] = entry.get('_time')
if 'index' in entry: entry.pop('index')
if 'sourcetype' in entry: entry.pop('sourcetype')
if 'source' in entry: entry.pop('source')
if 'host' in entry: entry.pop('host')
# event payload minus hidden fields and reserved splunk fields
entry = {
k: entry.get(k)
for k, v in entry.items() if k.startswith('_')
}
# clean off Splunk specific fields from payload
entry.pop('punct')
entry.pop('splunk_server')
payload['event'] = entry
destCollector.batchEvent(payload)
if not hasSearchResults:
helper.log_info("Empty Search Results")
else:
destCollector.flushBatch()
helper.log_info("Alert action sendtohec completed.")
return 0
import json
def commitCrime():
# list of sample values
suspects = ['Miss Scarett','Professor Plum','Miss Peacock','Mr. Green','Colonel Mustard','Mrs. White']
weapons = ['candlestick','knife','lead pipe','revolver','rope','wrench']
rooms = ['kitchen','ballroom','conservatory','dining room','cellar','billiard room','library','lounge','hall','study']
return {"killer":random.choice(suspects), "weapon":random.choice(weapons), "location":random.choice(rooms), "victim":"Mr Boddy"}
# Create event collector object, default SSL and HTTP Event Collector Port
http_event_collector_key = "B02336E2-EEC2-48FF-9FA8-267B553A0C6B"
http_event_collector_host = "localhost"
testevent = http_event_collector(http_event_collector_key, http_event_collector_host)
# Start event payload and add the metadata information
payload = {}
payload.update({"index":"temp"})
payload.update({"sourcetype":"crime"})
payload.update({"source":"witness"})
payload.update({"host":"mansion"})
# Report 5 Crimes
for i in range(5):
event = commitCrime()
event.update({"action":"success"})
event.update({"crime_type":"single"})
event.update({"crime_number":i})
payload.update({"event":event})
import requests
import datetime
from splunk_http_event_collector import http_event_collector
key = "7f75b2f2-2b0c-41c9-b9ac-beeec40f7410"
host = "localhost"
testevent = http_event_collector(key, host)
testevent.popNullFields = True
payload = {}
payload.update({"index": "main"})
payload.update({"sourcetype": "Temperature"})
payload.update({"source": "Apartment Temperature Reader"})
payload.update({"host": "Pi"})
payload.update({"event": {"Temp": 22.1}})
result = testevent.sendEvent(payload)
"killer": random.choice(suspects),
"weapon": random.choice(weapons),
"location": random.choice(rooms),
"victim": "Mr Boddy"
}
# init logging config, this would be job of your main code using this class.
logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s',
datefmt='%Y-%m-%d %H:%M:%S %z')
# Create event collector object, default SSL and HTTP Event Collector Port
http_event_collector_key = "4D14F8D9-D788-4E6E-BF2D-D1A46441242E"
http_event_collector_host = "localhost"
testevent = http_event_collector(http_event_collector_key,
http_event_collector_host)
# perform a HEC reachable check
hec_reachable, hec_conn_msg = testevent.check_connectivity()
if not hec_reachable:
print("Connection issue: {} (exiting)".format(hec_conn_msg))
sys.exit(1)
# Set to pop null fields. Always a good idea
testevent.popNullFields = True
# set logging to DEBUG for example
testevent.log.setLevel(logging.DEBUG)
# Start event payload and add the metadata information
payload = {}
payload.update({"index": "test"})
import os
import glob
import time
import logging
from splunk_http_event_collector import http_event_collector
splunkHECKey = "7f75b2f2-2b0c-41c9-b9ac-beeec40f7410"
splunkHECHost = "192.168.2.11"
eventSender = http_event_collector(splunkHECKey, splunkHECHost)
os.system('modprobe w1-gpio')
os.system('modprobe w1-therm')
base_dir = '/sys/bus/w1/devices/'
device_folder = glob.glob(base_dir + '28*')[0]
device_file = device_folder + '/w1_slave'
def read_temp_raw():
f = open(device_file, 'r')
lines = f.readlines()
f.close()
return lines
def read_temp():
lines = read_temp_raw()
while lines[0].strip()[-3:] != 'YES':
time.sleep(0.2)
lines = read_temp_raw()
equals_pos = lines[1].find('t=')
if equals_pos != -1:
def lambda_handler(event, context):
http_event_collector_token = os.environ['SPLUNK_HEC_TOKEN']
http_event_collector_host, http_event_collector_port, http_event_collector_path = hostIsolator(
os.environ['SPLUNK_HEC_HOST'])
eventSenderJSON = http_event_collector(
http_event_collector_token,
http_event_collector_host,
'json',
http_event_port=http_event_collector_port)
eventSenderJSON.debug = True
session = boto3.session.Session()
securityhub = session.client('securityhub',
region_name=os.environ['AWS_REGION'])
results = getFindings(session, securityhub)
fresh_events_after_this_time = freshEventTimestampGenerator()
fresh_events = True
first_call = True
sent_count = 0
while ((first_call or 'NextToken' in results) and fresh_events):
# Loop through all findings (20 by default) returned by Security Hub API call
# If finding has the string "SENT TO SPLUNK" in the finding note, the event is not sent but
# loop will continue.
# Fresh events will be sent to Splunk over HTTP Event Collector (HEC), "SENT TO SPLUNK" will
# be prefixed to the finding's note.
# Break out of the loop when we have looked back across the last hour of events (based on the
# finding's LastObservedAt timestamp)
first_call = False
for finding in results['Findings']:
finding_timestamp = findingTimestampGenerator(
finding['LastObservedAt'])
already_sent = False
existing_note = ''
principal = 'SplunkSecurityHubLambda'
if 'Note' in finding:
if 'SENT TO SPLUNK:' in finding['Note']['Text']:
already_sent = True
else:
existing_note = finding['Note']['Text']
principal = finding['Note']['UpdatedBy']
if (finding_timestamp > fresh_events_after_this_time
and not already_sent):
payload = {}
payload.update({'sourcetype': 'aws:securityhub'})
payload.update({'event': json.dumps(finding)})
filters = {
'Id': [{
'Comparison': 'EQUALS',
'Value': finding['Id']
}],
'LastObservedAt': [{
'Start': finding['LastObservedAt'],
'End': finding['LastObservedAt']
}],
}
eventSenderJSON.sendEvent(payload)
if not eventSenderJSON.failedToSend:
print('Event successfully sent to Splunk')
eventSenderJSON.failedToSend = False
updateFindingNote(session, securityhub, existing_note,
principal, filters)
sent_count += 1
else:
print('Event NOT successfully sent to Splunk')
else:
fresh_events = False
break
if (fresh_events):
results = getFindingsWithToken(session, securityhub,
results['NextToken'])
print '%s findings sent to Splunk: %s' % (sent_count,
eventSenderJSON.server_uri)
from subprocess import PIPE, Popen, check_output
from PIL import Image, ImageDraw, ImageFont
from fonts.ttf import RobotoMedium as UserFont
from enviroplus import gas
from splunk_http_event_collector import http_event_collector
import socket
import os
try:
from smbus2 import SMBus
except ImportError:
from smbus import SMBus
# setup send to splunk
try:
events = http_event_collector(os.getenv("HEC_TOKEN"), "192.168.99.60")
except Exception as e:
print(f"missing env key e={e}")
sys.exit(-1)
hec_payload = {}
bus = SMBus(1)
# Create BME280 instance
bme280 = BME280(i2c_dev=bus)
# Create LCD instance
disp = ST7735.ST7735(port=0,
cs=1,
dc=9,
backlight=12,
try:
from splunk_http_event_collector import http_event_collector #type:ignore
except ImportError as error_message:
sys.exit(f"Failed to import splunk_http_event_collector: {error_message}")
from . import API
if __name__ == '__main__':
if not os.getenv("PARENT_HOST"):
sys.exit("PARENT_HOST environment variable is not set, bailing.")
api = API()
hec = http_event_collector(
token=os.getenv('HECTOKEN'),
http_event_server=os.getenv('HECHOST'),
http_event_port=os.getenv('HECPORT', "443"),
http_event_server_ssl=True,
)
hec.index = os.getenv("HECINDEX", "torrent")
hec.log.setLevel(logging.DEBUG)
# pylint: disable=invalid-name
queue_counter = 0
hostname = gethostname()
for torrent in api.get_torrents():
# add this so we can troubleshoot later
torrent["CONTAINER_ID"] = hostname
payload = {
"sourcetype": os.getenv('HECSOURCETYPE', "torrent:info"),
def __init__(self):
super().__init__()
self.HEC_sender = http_event_collector(self.http_event_collector_key,
self.http_event_collector_host)
def Output(hashes_list, stats):
if debug: print("\n\nMODULE OUTPUT")
if debug: print("FUNCTION output.Output")
details = ""
vt_malware_threshold = int(config.get('config', 'vt_malware_threshold'))
#Setup output options
#Local
local = config.get('config', 'local')
if local == 'yes':
history = open('history.csv', 'a')
size = os.path.getsize('history.csv')
#If new file, output header
if size == 0:
output = "timestamp,environment,computer,file,path,size,md5,sha256,Source,wf_malware,wf_new,wf_upload,vt_positive,vt_total,vt_link,vt_new,vt_upload\n"
history.write(output)
if debug: print(" Will output to local file")
#Splunk
splunk = config.get('config', 'splunk')
if splunk == 'yes':
http_event_collector_key = config.get('config', 'splunk_key')
http_event_collector_host = config.get('config', 'splunk_host')
event = http_event_collector(http_event_collector_key,
http_event_collector_host)
payload = {}
payload.update({"index": config.get('config', 'splunk_index')})
payload.update(
{"sourcetype": config.get('config', 'splunk_sourcetype')})
payload.update({"source": config.get('config', 'splunk_source')})
if debug: print(" Will output to Splunk")
#Slack
slack = config.get('config', 'slack')
if slack == 'yes':
token = config.get('config', 'slack_token')
username = config.get('config', 'slack_username')
channel = config.get('config', 'slack_channel')
if debug: print(" Will output to Slack")
#Email
email = config.get('config', 'email')
email_attachments = []
if debug and email == 'yes': print(" Will output to Email")
now = datetime.datetime.now()
timestamp = str(now.year) + '-' + '{:02d}'.format(
now.month) + '-' + '{:02d}'.format(now.day) + ' ' + str(
now.hour) + ':' + str(now.minute)
#Output results to selected options
for hash in hashes_list:
computer = hash[0]
file = hash[1]
path = hash[2]
size = hash[3]
md5 = hash[4]
sha256 = hash[5]
source = hash[6]
wf_malware = str(hash[7])
wf_new = str(hash[8])
wf_upload = str(hash[9])
vt_positive = str(hash[10])
vt_total = str(hash[11])
vt_link = hash[12]
vt_new = str(hash[13])
vt_upload = str(hash[14])
#Count malware hashes
if wf_malware == 'yes' or int(vt_positive) >= vt_malware_threshold:
stats['malware'] += 1
#Output to CSV
if local == 'yes':
output = timestamp + ',' + environment + ',' + computer + ',' + file + ',' + path + ',' + size + ',' + md5 + ',' + sha256 + ',' + source + ',' + wf_malware + ',' + wf_new + ',' + wf_upload + ',' + vt_positive + ',' + vt_total + ',' + vt_link + ',' + vt_new + ',' + vt_upload + '\n'
history.write(output)
#Output to Splunk
if splunk == 'yes':
event_data = {
'timestamp': timestamp,
'environment': environment,
'type': 'detail',
'hash_source': 'index',
'computer': computer,
'file': file,
'path': path,
'size': size,
'md5': md5,
'sha256': sha256,
'source': source,
'wf_malware': wf_malware,
'wf_new': wf_new,
'wf_upload': wf_upload,
'vt_positive': vt_positive,
'vt_total': vt_total,
'vt_link': vt_link,
'vt_new': vt_new,
'vt_upload': vt_upload
}
payload.update({'event': event_data})
event.batchEvent(payload)
#Output to Slack only if malware
if slack == 'yes' and (wf_malware == 'yes'
or int(vt_positive) >= vt_malware_threshold):
text = ("MALWARE FOUND" + \
"\nTimestamp: " + timestamp + \
"\nComputer: " + computer + \
"\nFile: " + file + \
"\nPath: " + path + \
"\nMD5: " + md5 + \
"\nSHA256: " + sha256 + \
"\nWildFire Malware: " + wf_malware + \
"\nVirusTotal: " + str(vt_positive) + '/' + str(vt_total) + \
"\nVirusTotal Link: " + vt_link)
message = urllib2.quote(text)
#results = 'emptyStatus'
try:
url = "https://slack.com/api/chat.postMessage?token=" + token + "&channel=" + channel + "&text=" + message + "&username="******"&pretty=1"
req = urllib2.Request(url)
response = urllib2.urlopen(req)
results = response.read()
except (urllib2.HTTPError) as e:
results = e
if wf_malware == 'yes' or int(vt_positive) >= vt_malware_threshold:
details += "\nComputer: " + computer
details += "\nFile: " + file
details += "\nPath: " + path
details += "\nMD5: " + md5
details += "\nSHA256: " + sha256
details += "\nWildFire malware: " + wf_malware
details += "\nVirusTotal: " + str(vt_positive) + '/' + str(
vt_total)
details += "\nVirusTotal link: " + vt_link + '\n'
email_attachments.append("reports\\" + md5 + ".pdf")
#Output Final statistics and details
computers_total = str(stats['computers_total'])
computers_hashes = str(stats['computers_hashes'])
total = str(stats['total'])
excluded = str(stats['excluded'])
unique = str(stats['unique'])
wf_cache = str(stats['wf_cache'])
wf_new = str(stats['wf_new'])
wf_uploaded = str(stats['wf_uploaded'])
vt_cache = str(stats['vt_cache'])
vt_new = str(stats['vt_new'])
vt_uploaded = str(stats['vt_uploaded'])
malware = str(stats['malware'])
email_content = "\nTanFire Statistics" + \
"\nComputers (total/with hashes): " + computers_total + '/' + computers_hashes + \
"\nNew Tanium Index hashes: " + total + \
"\nExcluded: " + excluded + \
"\nUnique: " + unique + \
"\nWildFire cache: " + wf_cache + \
"\nWildFire direct: " + wf_new + \
"\nWildFire uploaded: " + wf_uploaded + \
"\nVirusTotal cache: " + vt_cache + \
"\nVirusTotal direct: " + vt_new + \
"\nMalware: " + malware + \
"\n\nDetails on malware hashes: " + details
if debug:
print(email_content)
if local == 'yes':
stats_file = open('stats.csv', 'a')
size = os.path.getsize('stats.csv')
#If new file, output header
if size == 0:
output = "timestamp,type,computers_total,computers_hashes,total,excluded,unique,wf_cache,wf_new,wf_ploaded,vt_cache,vt_new,vt_uploaded,malware\n"
stats_file.write(output)
output = timestamp + ',index,' + computers_total + ',' + computers_hashes + ',' + total + ',' + excluded + ',' + unique + ',' + wf_cache + ',' + wf_new + ',' + wf_uploaded + ',' + vt_cache + ',' + vt_new + ',' + vt_uploaded + ',' + malware + '\n'
stats_file.write(output)
stats_file.close()
if splunk == 'yes':
event_data = {
'timestamp': timestamp,
'type': 'summary',
'environment': environment,
'hash_source': 'index',
'computers_total': computers_total,
'computers_hashes': computers_hashes,
'total': total,
'excluded': excluded,
'unique': unique,
'wf_cache': wf_cache,
'wf_new': wf_new,
'wf_uploaded': wf_uploaded,
'vt_cache': vt_cache,
'vt_new': vt_new,
'vt_uploaded': vt_uploaded,
'malware': malware
}
payload.update({'event': event_data})
event.batchEvent(payload)
#Output to Email
if email == 'yes': Email(timestamp, email_content, email_attachments)
#Output cleanup
if local == 'yes': history.close()
if splunk == 'yes': event.flushBatch()
return {
"killer": random.choice(suspects),
"weapon": random.choice(weapons),
"location": random.choice(rooms),
"victim": "Mr Boddy"
}
# Create event collector object, default SSL and HTTP Event Collector Port
http_event_collector_key = "DB84F19F-B2F1-4B89-BB38-643DFB641B34"
http_event_collector_host = "45.55.161.5"
testevent = http_event_collector(http_event_collector_key,
http_event_collector_host,
input_type='json',
host="",
http_event_port='8088',
http_event_server_ssl=True,
http_event_collector_debug=True)
# Start event payload and add the metadata information
payload = {}
payload.update({"index": "http-test"})
payload.update({"sourcetype": "crime"})
payload.update({"source": "witness"})
payload.update({"host": "mansion"})
# Report 5 Crimes
for i in range(5):
event = commitCrime()
event.update({"action": "success"})
