def onMessage(self, message, metadata):
# check for category like 'ldap' and rename the tls field
if key_exists('category', message):
data = message.get('category')
if data == 'ldap':
if key_exists('details.tls', message):
message['details']['tls_encrypted'] = message['details'][
'tls']
del (message['details']['tls'])
if 'source' not in message:
message['source'] = 'ldap'
details = message.get('details', {})
actor_str = details.get('actor', '')
actor_email = _parse_email_from_actor(actor_str)
if actor_email is None:
details['email'] = None
details['username'] = None
message['details'] = details
return (message, metadata)
username = actor_email.split('@')[0]
details['email'] = actor_email
details['username'] = username
message['details'] = details
return (message, metadata)
def onMessage(self, message, metadata):
if 'source' not in message:
return (message, metadata)
if not message['source'] == 'guardduty':
return (message, metadata)
# reformat the date fields to iosformat
for date_key in self.date_keys:
if key_exists(date_key, message):
message = self.convert_key_date_format(date_key, message)
# convert the dict to a dot dict for saner deep key/value processing
message = DotDict(message)
# pull out the likely source IP address
for ipaddress_key in self.ipaddress_keys:
if 'sourceipaddress' not in message['details'].keys():
if key_exists(ipaddress_key, message):
message.details.sourceipaddress = message.get(
ipaddress_key)
# if we still haven't found what we are looking for #U2
# sometimes it's in a list
if 'sourceipaddress' not in message['details'].keys():
if key_exists('details.finding.action.portProbeAction.portProbeDetails', message) \
and isinstance(message.details.finding.action.portProbeAction.portProbeDetails, list):
# inspect the first list entry and see if it contains an IP
portProbeDetails = DotDict(message.details.finding.action.
portProbeAction.portProbeDetails[0])
if key_exists('remoteIpDetails.ipAddressV4', portProbeDetails):
message.details.sourceipaddress = portProbeDetails.remoteIpDetails.ipAddressV4
# recovert the message back to a plain dict
return (dict(message), metadata)
def onMessage(self, message, metadata):
# check for category like 'ldap' and rename the tls field
if key_exists('category', message):
data = message.get('category')
if data == 'ldap':
if key_exists('details.tls', message):
message['details']['tls_encrypted'] = message['details'][
'tls']
del (message['details']['tls'])
if 'source' not in message:
message['source'] = 'ldap'
return (message, metadata)
def on_message(self, message_raw):
if "Message" in message_raw:
message = json.loads(message_raw["Message"])
if key_exists('details.finding.action.actionType', message):
if message["details"]["finding"]["action"]["actionType"] == "PORT_PROBE":
if "portProbeDetails" in message["details"]["finding"]["action"]["portProbeAction"]:
for probe in message["details"]["finding"]["action"]["portProbeAction"]["portProbeDetails"]:
isolatedmessage = message
isolatedmessage["details"]["finding"]["probeevent"] = probe
self.build_submit_message(isolatedmessage)
elif message["details"]["finding"]["action"]["actionType"] == "AWS_API_CALL":
# Fixup the api call data somewhat, depending on what we were given
if "recentApiCalls" in message["details"]["finding"]["additionalInfo"]:
message["details"]["finding"]["additionalInfo"]["apiCalls"] = message["details"]["finding"][
"additionalInfo"
]["recentApiCalls"]
if "apiCalls" not in message["details"]["finding"]["additionalInfo"]:
message["details"]["finding"]["additionalInfo"]["apiCalls"] = message["details"]["finding"][
"action"
]["awsApiCallAction"]
if type(message["details"]["finding"]["additionalInfo"]["apiCalls"]) == list:
for call in message["details"]["finding"]["additionalInfo"]["apiCalls"]:
isolatedmessage = message
isolatedmessage["details"]["finding"]["apicalls"] = call
self.build_submit_message(isolatedmessage)
else:
isolatedmessage = message
isolatedmessage["details"]["finding"]["apicalls"] = message["details"]["finding"][
"additionalInfo"
]["apiCalls"]
self.build_submit_message(isolatedmessage)
else:
self.build_submit_message(message)
def onMessage(self, message, metadata):
'''
Check if source is in the message, if not then add cloudtrail as the source.
'''
if 'source' not in message:
return (message, metadata)
if not message['source'] == 'cloudtrail':
return (message, metadata)
'''
Check if details.requestparameters.htmlpart exists, if it does it's generally longer than 32000 bytes,
so we'll truncate it to 4096 characters using the constant ES_FIELD_VALUE_LIMIT so that ES will ingest it,
leaving us with knowledge of what the field contains without the overkill of storing the entire page.
'''
ES_FIELD_VALUE_LIMIT = 4095
if 'requestparameters' in message['details'] and message['details']['requestparameters']:
if 'htmlpart' in message['details']['requestparameters'] and message['details']['requestparameters']['htmlpart']:
message['details']['requestparameters']['htmlpart'] = message['details']['requestparameters']['htmlpart'][0:ES_FIELD_VALUE_LIMIT]
for modified_key in self.modify_keys:
if key_exists(modified_key, message):
message = self.convert_key_raw_str(modified_key, message)
return (message, metadata)
def on_message(self, message_raw):
if "Message" in message_raw:
message = json.loads(message_raw["Message"])
if key_exists('details.finding.action.actionType', message):
if message["details"]["finding"]["action"][
"actionType"] == "PORT_PROBE":
if "portProbeDetails" in message["details"]["finding"][
"action"]["portProbeAction"]:
for probe in message["details"]["finding"]["action"][
"portProbeAction"]["portProbeDetails"]:
isolatedmessage = message
isolatedmessage["details"]["finding"][
"probeevent"] = probe
self.build_submit_message(isolatedmessage)
elif message["details"]["finding"]["action"][
"actionType"] == "AWS_API_CALL":
if "recentApiCalls" in message["details"]["finding"][
"additionalInfo"]:
message["details"]["finding"]["additionalInfo"][
"apiCalls"] = message["details"]["finding"][
"additionalInfo"]["recentApiCalls"]
for call in message["details"]["finding"][
"additionalInfo"]["apiCalls"]:
isolatedmessage = message
isolatedmessage["details"]["finding"][
"apicalls"] = call
self.build_submit_message(isolatedmessage)
else:
self.build_submit_message(message)
def onMessage(self, message, metadata):
if key_exists('tags', message):
if 'zoom' not in message['tags']:
return (message, metadata)
newmessage = {}
newmessage['details'] = {}
newmessage['category'] = 'zoom'
newmessage['eventsource'] = 'MozDef-EF-zoom'
newmessage['source'] = 'api_aws_lambda'
newmessage['hostname'] = 'zoom_host'
newmessage['severity'] = 'info'
newmessage['processname'] = 'zoom_webhook_api'
if key_exists('tags', message):
newmessage['tags'] = message['tags']
if key_exists('details.event', message):
newmessage['details']['event'] = message['details']['event']
else:
newmessage['details']['event'] = 'UNKNOWN'
if key_exists('details.payload.account_id', message):
newmessage['details']['account_id'] = message['details']['payload']['account_id']
if key_exists('details.payload.object.account_id', message):
if newmessage.get('details.account_id') != message.get('details.payload.object.account_id'):
newmessage['details']['meeting_account_id'] = message['details']['payload']['object']['account_id']
elif key_exists('details.payload.object.account_id', message):
newmessage['details']['account_id'] = message['details']['payload']['object']['account_id']
# rewrite summary to be more informative
newmessage['summary'] = ""
if key_exists('details.event', newmessage):
newmessage['summary'] = "zoom: {0}".format(message['details']['event'])
if key_exists('details.payload.object.participant.user_name', message):
newmessage['summary'] += " triggered by user {0}".format(message['details']['payload']['object']['participant']['user_name'])
elif key_exists('details.payload.operator', message):
newmessage['summary'] += " triggered by user {0}".format(message['details']['payload']['operator'])
# iterate through top level keys - push, etc
if newmessage['source'] in self.eventtypes:
for key in self.yap[newmessage['source']]:
mappedvalue = jmespath.search(self.yap[newmessage['source']][key], message)
# JMESPath likes to silently return a None object
if mappedvalue is not None:
newmessage['details'][key] = mappedvalue
else:
newmessage = None
return (newmessage, metadata)
def onMessage(self, message, metadata):
if 'source' not in message:
return (message, metadata)
if not message['source'] == 'cloudtrail':
return (message, metadata)
for modified_key in self.modify_keys:
if key_exists(modified_key, message):
message = self.convert_key_raw_str(modified_key, message)
return (message, metadata)
def onMessage(self, message, metadata):
# check for messages we have vetted as n/a and prevalent
# from a sec standpoint and drop them also rewrite fields
# to drop unecessary expansion
# omit "topic" field
if key_exists('details.payload.object.topic', message):
del message['details']['payload']['object']['topic']
# rewrite summary to be more informative
message['summary'] = ""
if key_exists('details.event', message):
message['summary'] = "zoom: {0}".format(
message['details']['event'])
if key_exists('details.payload.object.participant.user_name',
message):
message['summary'] += " triggered by user {0}".format(
message['details']['payload']['object']['participant']
['user_name'])
elif key_exists('details.payload.operator', message):
message['summary'] += " triggered by user {0}".format(
message['details']['payload']['operator'])
# drop duplicated account_id field
if key_exists('details.payload.account_id', message) and key_exists(
'details.payload.object.account_id', message):
if message.get('details.payload.account_id') == message.get(
'details.payload.object.account_id'):
del message['details']['payload']['object']['account_id']
return (message, metadata)
def onMessage(self, message, metadata):
if 'source' not in message:
return (message, metadata)
if not message['source'] == 'guardduty':
return (message, metadata)
# reformat the date fields to iosformat
for date_key in self.date_keys:
if key_exists(date_key, message):
if message.get(date_key) is None:
continue
else:
message = self.convert_key_date_format(date_key, message)
# convert the dict to a dot dict for saner deep key/value processing
message = DotDict(message)
# pull out the likely source IP address
for ipaddress_key in self.ipaddress_keys:
if 'sourceipaddress' not in message['details']:
if key_exists(ipaddress_key, message):
message.details.sourceipaddress = message.get(
ipaddress_key)
# if we still haven't found what we are looking for #U2
# sometimes it's in a list
if 'sourceipaddress' not in message['details']:
if key_exists('details.finding.action.portprobeaction.portprobedetails', message) \
and isinstance(message.details.finding.action.portprobeaction.portprobedetails, list):
# inspect the first list entry and see if it contains an IP
portprobedetails = DotDict(
message.details.finding.action.portprobeaction.portprobedetails[0])
if key_exists('remoteipdetails.ipaddressv4', portprobedetails):
message.details.sourceipaddress = portprobedetails.remoteipdetails.ipaddressv4
# recovert the message back to a plain dict
return (dict(message), metadata)
def test_basic_dict_positive_complicated_2(self):
assert key_exists('details.depth2.depth1', self.default_event) is False
def test_basic_dict_positive_complicated(self):
assert key_exists('details.depth1.depth2.depth3.somekey',
self.default_event) is True
def test_basic_dict_negative(self):
assert key_exists('otherkey', {'details': 'abcd'}) is False
def test_basic_dict_positive(self):
assert key_exists('details', {'details': 'abcd'}) is True
def test_basic_dict_positive_complicated(self):
assert key_exists('details.depth1.depth2.depth3.somekey', self.default_event) is True
def test_basic_dict_positive(self):
assert key_exists('details', {'details': 'abcd'}) is True
def onMessage(self, message, metadata):
if key_exists('tags', message):
if 'zoom' not in message['tags']:
return (message, metadata)
newmessage = {}
newmessage['details'] = {}
newmessage['category'] = 'zoom'
newmessage['eventsource'] = 'MozDef-EF-zoom'
newmessage['source'] = 'api_aws_lambda'
newmessage['hostname'] = 'zoom_host'
newmessage['severity'] = 'info'
newmessage['processname'] = 'zoom_webhook_api'
if key_exists('tags', message):
newmessage['tags'] = message['tags']
if key_exists('details.event', message):
newmessage['details']['event'] = message['details']['event']
else:
newmessage['details']['event'] = 'UNKNOWN'
if key_exists('details.payload.account_id', message):
newmessage['details']['account_id'] = message['details'][
'payload']['account_id']
if key_exists('details.payload.object.account_id', message):
if newmessage.get('details.account_id') != message.get(
'details.payload.object.account_id'):
newmessage['details']['meeting_account_id'] = message[
'details']['payload']['object']['account_id']
elif key_exists('details.payload.object.account_id', message):
newmessage['details']['account_id'] = message['details'][
'payload']['object']['account_id']
# rewrite summary to be more informative
newmessage['summary'] = ""
if key_exists('details.event', newmessage):
newmessage['summary'] = "zoom: {0}".format(
message['details']['event'])
if key_exists('details.payload.object.participant.user_name',
message):
newmessage['summary'] += " triggered by user {0}".format(
message['details']['payload']['object']['participant']
['user_name'])
elif key_exists('details.payload.operator', message):
newmessage['summary'] += " triggered by user {0}".format(
message['details']['payload']['operator'])
# iterate through top level keys - push, etc
if newmessage['source'] in self.eventtypes:
for key in self.yap[newmessage['source']]:
mappedvalue = jmespath.search(
self.yap[newmessage['source']][key], message)
# JMESPath likes to silently return a None object
if mappedvalue is not None:
newmessage['details'][key] = mappedvalue
# Some zoom messages don't contain values within details.payload.object.start_time or details.payload.old_object.start_time.
# so we set it to original start time, if there is no time data, we remove the key from the message.
if key_exists('details.payload.object.start_time', message):
if message['details']['payload']['object']['start_time'] != '':
newmessage['details']['start_time'] = message['details'][
'payload']['object']['start_time']
else:
del newmessage['details']['start_time']
if key_exists('details.payload.old_object.start_time', message):
if message['details']['payload']['old_object'][
'start_time'] != '':
newmessage['details'][
'original_sched_start_time'] = message['details'][
'payload']['old_object']['start_time']
else:
del newmessage['details']['original_sched_start_time']
# Duration can exist in details.payload.object and details.payload.old_object, let's ensure we are capturing these correctly for updated meetings.
if key_exists('details.payload.object.duration', message):
newmessage['details']['duration'] = message['details'][
'payload']['object']['duration']
if key_exists('details.payload.old_object.duration', message):
newmessage['details']['original_sched_duration'] = message[
'details']['payload']['old_object']['duration']
else:
newmessage = None
return (newmessage, metadata)
def onMessage(self, message, metadata):
if 'tags' not in message:
return (message, metadata)
if 'githubeventsqs' not in message['tags']:
return (message, metadata)
newmessage = {}
newmessage['details'] = {}
newmessage['category'] = 'github'
newmessage['tags'] = ['github', 'webhook']
newmessage['eventsource'] = 'githubeventsqs'
if key_exists('details.event', message):
newmessage['source'] = message['details']['event']
else:
newmessage['source'] = 'UNKNOWN'
if key_exists('details.request_id', message):
newmessage['details']['request_id'] = message['details']['request_id']
else:
newmessage['details']['request_id'] = 'UNKNOWN'
# iterate through top level keys - push, etc
if newmessage['source'] in self.eventtypes:
for key in self.yap[newmessage['source']]:
mappedvalue = jmespath.search(self.yap[newmessage['source']][key], message)
# JMESPath likes to silently return a None object
if mappedvalue is not None:
newmessage['details'][key] = mappedvalue
if 'commit_ts' in newmessage['details']:
newmessage['timestamp'] = newmessage['details']['commit_ts']
newmessage['utctimestamp'] = toUTC(newmessage['details']['commit_ts']).isoformat()
else:
newmessage = None
if key_exists('source', newmessage) and newmessage.get('source') is not 'UNKNOWN':
newmessage['summary'] = "github: {0}: ".format(newmessage['source'])
if key_exists('source', newmessage) and newmessage.get('source') is 'installation':
newmessage['summary'] = "github app: {0} ".format(newmessage['source'])
if key_exists('source', newmessage) and newmessage.get('source') is 'public':
newmessage['summary'] = "github : change from private to {0} ".format(newmessage['source'])
if key_exists('details.status', newmessage):
action_status = "{0} ".format(newmessage['details']['status'])
newmessage['summary'] += action_status
if key_exists('details.action', newmessage):
github_action = "{0} ".format(newmessage['details']['action'])
newmessage['summary'] += github_action
if key_exists('details.ref_type', newmessage):
reference = "{0} ".format(newmessage['details']['ref_type'])
newmessage['summary'] += reference
if key_exists('details.repo_name', newmessage):
repository_name = "on repo: {0} ".format(newmessage['details']['repo_name'])
newmessage['summary'] += repository_name
if key_exists('details.alert_note', newmessage):
sec_advisory = "for: {0}".format(newmessage['details']['alert_note'])
newmessage['summary'] += sec_advisory
if key_exists('details.alert_package', newmessage):
vuln_package = "package: {0} ".format(newmessage['details']['alert_package'])
newmessage['summary'] += vuln_package
if key_exists('details.team_name', newmessage):
team_name = "team: {0} ".format(newmessage['details']['team_name'])
newmessage['summary'] += team_name
if key_exists('details.blocked_user_login', newmessage):
blocked_user = "******".format(newmessage['details']['blocked_user_login'])
newmessage['summary'] += blocked_user
if key_exists('details.org_login', newmessage):
org_name = "in org: {0} ".format(newmessage['details']['org_login'])
newmessage['summary'] += org_name
if key_exists('details.username', newmessage):
github_user = "******".format(newmessage['details']['username'])
newmessage['summary'] += "triggered by " + github_user
return (newmessage, metadata)
def onMessage(self, message, metadata):
# make sure I really wanted to see this message
# bail out early if not
if 'customendpoint' not in message:
return message, metadata
if 'category' not in message:
return message, metadata
if 'SOURCE' not in message:
return message, metadata
if message['category'] != 'bro':
return message, metadata
# move Bro specific fields under 'details' while preserving metadata
newmessage = dict()
# default replacement for old _type subcategory.
# to preserve filtering capabilities
newmessage['type']= 'nsm'
try:
newmessage['details'] = json.loads(message['MESSAGE'])
except:
newmessage['details'] = {}
newmessage['rawdetails'] = message
newmessage['customendpoint'] = 'bro'
# move some fields that are expected at the event 'root' where they belong
if 'HOST_FROM' in message:
newmessage['hostname'] = message['HOST_FROM']
if 'tags' in message:
newmessage['tags'] = message['tags']
if 'category' in message:
newmessage['category'] = message['category']
if 'SOURCE' in message:
# transform bro_files into files fast
newmessage['source'] = message['SOURCE'][4:]
if 'resp_cc' in newmessage['details']:
del(newmessage['details']['resp_cc'])
# add mandatory fields
if 'ts' in newmessage['details']:
newmessage['utctimestamp'] = toUTC(float(newmessage['details']['ts'])).isoformat()
newmessage['timestamp'] = toUTC(float(newmessage['details']['ts'])).isoformat()
# del(newmessage['details']['ts'])
else:
# a malformed message somehow managed to crawl to us, let's put it somewhat together
newmessage['utctimestamp'] = toUTC(datetime.now()).isoformat()
newmessage['timestamp'] = toUTC(datetime.now()).isoformat()
newmessage['receivedtimestamp'] = toUTC(datetime.now()).isoformat()
newmessage['eventsource'] = 'nsm'
newmessage['severity'] = 'INFO'
newmessage['mozdefhostname'] = self.mozdefhostname
if 'id.orig_h' in newmessage['details']:
newmessage['details']['sourceipaddress'] = newmessage['details']['id.orig_h']
del(newmessage['details']['id.orig_h'])
if 'id.orig_p' in newmessage['details']:
newmessage['details']['sourceport'] = newmessage['details']['id.orig_p']
del(newmessage['details']['id.orig_p'])
if 'id.resp_h' in newmessage['details']:
newmessage['details']['destinationipaddress'] = newmessage['details']['id.resp_h']
del(newmessage['details']['id.resp_h'])
if 'id.resp_p' in newmessage['details']:
newmessage['details']['destinationport'] = newmessage['details']['id.resp_p']
del(newmessage['details']['id.resp_p'])
if 'details' in newmessage:
if 'FILE_NAME' in newmessage['details']:
del(newmessage['details']['FILE_NAME'])
if 'MESSAGE' in newmessage['details']:
del(newmessage['details']['MESSAGE'])
if 'SOURCE' in newmessage['details']:
del(newmessage['details']['SOURCE'])
# All Bro logs need special treatment, so we provide it
# Not a known log source? Mark it as such and return
if 'source' not in newmessage:
newmessage['source'] = 'unknown'
return newmessage, metadata
else:
logtype = newmessage['source']
if logtype == 'conn':
newmessage['details']['originipbytes'] = newmessage['details']['orig_ip_bytes']
newmessage['details']['responseipbytes'] = newmessage['details']['resp_ip_bytes']
del(newmessage['details']['orig_ip_bytes'])
del(newmessage['details']['resp_ip_bytes'])
if 'history' not in newmessage['details']:
newmessage['details']['history'] = ''
newmessage['summary'] = (
'{sourceipaddress}:'+
'{sourceport} -> '+
'{destinationipaddress}:'
'{destinationport} '+
'{history} '+
'{originipbytes} bytes / '
'{responseipbytes} bytes'
).format(**newmessage['details'])
return (newmessage, metadata)
if logtype == 'files':
if 'rx_hosts' in newmessage['details']:
newmessage['details']['sourceipaddress'] = '{0}'.format(newmessage['details']['rx_hosts'][0])
if 'tx_hosts' in newmessage['details']:
newmessage['details']['destinationipaddress'] = '{0}'.format(newmessage['details']['tx_hosts'][0])
if 'mime_type' not in newmessage['details']:
newmessage['details']['mime_type'] = 'unknown'
if 'filename' not in newmessage['details']:
newmessage['details']['filename'] = 'unknown'
if 'total_bytes' not in newmessage['details']:
newmessage['details']['total_bytes'] = '0'
if 'md5' not in newmessage['details']:
newmessage['details']['md5'] = 'None'
if 'filesource' not in newmessage['details']:
newmessage['details']['filesource'] = 'None'
newmessage['summary'] = (
'{rx_hosts[0]} '
'downloaded (MD5) '
'{md5} '
'MIME {mime_type} '
'({total_bytes} bytes) '
'from {tx_hosts[0]} '
'via {filesource}'
).format(**newmessage['details'])
return (newmessage, metadata)
if logtype == 'dns':
if 'qtype_name' not in newmessage['details']:
newmessage['details']['qtype_name'] = 'unknown'
if 'query' not in newmessage['details']:
newmessage['details']['query'] = ''
if 'rcode_name' not in newmessage['details']:
newmessage['details']['rcode_name'] = ''
newmessage['summary'] = (
'DNS {qtype_name} type query '
'{sourceipaddress} -> '
'{destinationipaddress}:{destinationport}'
).format(**newmessage['details'])
return (newmessage, metadata)
if logtype == 'http':
if 'method' not in newmessage['details']:
newmessage['details']['method'] = ''
if 'host' not in newmessage['details']:
newmessage['details']['host'] = ''
if 'uri' not in newmessage['details']:
newmessage['details']['uri'] = ''
newmessage['details']['uri'] = newmessage['details']['uri'][:1024]
if 'status_code' not in newmessage['details']:
newmessage['details']['status_code'] = ''
newmessage['summary'] = (
'HTTP {method} '
'{sourceipaddress} -> '
'{destinationipaddress}:'
'{destinationport}'
).format(**newmessage['details'])
return (newmessage, metadata)
if logtype == 'ssl':
if 'server_name' not in newmessage['details']:
# fake it till you make it
newmessage['details']['server_name'] = newmessage['details']['destinationipaddress']
newmessage['summary'] = (
'SSL: {sourceipaddress} -> '
'{destinationipaddress}:'
'{destinationport}'
).format(**newmessage['details'])
return (newmessage, metadata)
if logtype == 'dhcp':
if 'assigned_addr' not in newmessage['details']:
newmessage['details']['assigned_addr'] = "0.0.0.0"
if 'mac' not in newmessage['details']:
newmessage['details']['mac'] = "000000000000"
newmessage['details']['mac'] = newmessage['details']['mac'].replace(':', '')
newmessage['summary'] = (
'{assigned_addr} assigned to '
'{mac}'
).format(**newmessage['details'])
return (newmessage, metadata)
if logtype == 'ftp':
if 'command' not in newmessage['details']:
newmessage['details']['command'] = ''
if 'user' not in newmessage['details']:
newmessage['details']['user'] = ''
newmessage['summary'] = (
'FTP: {sourceipaddress} -> '
'{destinationipaddress}:'
'{destinationport}'
).format(**newmessage['details'])
return (newmessage, metadata)
if logtype == 'pe':
if 'os' not in newmessage['details']:
newmessage['details']['os'] = ''
if 'subsystem' not in newmessage['details']:
newmessage['details']['subsystem'] = ''
newmessage['summary'] = (
'PE file: {os} '
'{subsystem}'
).format(**newmessage['details'])
return (newmessage, metadata)
if logtype == 'smtp':
newmessage['summary'] = (
'SMTP: {sourceipaddress} -> '
'{destinationipaddress}:'
'{destinationport}'
).format(**newmessage['details'])
if key_exists('details.tls', newmessage):
newmessage['details']['tls_encrypted'] = newmessage['details']['tls']
del(newmessage['details']['tls'])
return (newmessage, metadata)
if logtype == 'ssh':
summary = (
'SSH: {sourceipaddress} -> '
'{destinationipaddress}:'
'{destinationport}'
).format(**newmessage['details'])
if 'auth_success' in newmessage['details']:
summary += ' success {0}'.format(newmessage['details']['auth_success'])
newmessage['summary'] = summary
return (newmessage, metadata)
if logtype == 'tunnel':
if 'tunnel_type' not in newmessage['details']:
newmessage['details']['tunnel_type'] = ''
if 'action' not in newmessage['details']:
newmessage['details']['action'] = ''
newmessage['summary'] = (
'{sourceipaddress} -> '
'{destinationipaddress}:'
'{destinationport} '
'{tunnel_type} '
'{action}'
).format(**newmessage['details'])
return (newmessage, metadata)
if logtype == 'intel':
if 'seen.indicator' in newmessage['details']:
newmessage['details']['seenindicator'] = newmessage['details']['seen.indicator']
del(newmessage['details']['seen.indicator'])
else:
newmessage['details']['seenindicator'] = ''
if 'seen.node' in newmessage['details']:
newmessage['details']['seennode'] = newmessage['details']['seen.node']
del(newmessage['details']['seen.node'])
if 'seen.where' in newmessage['details']:
newmessage['details']['seenwhere'] = newmessage['details']['seen.where']
del(newmessage['details']['seen.where'])
if 'seen.indicator_type' in newmessage['details']:
newmessage['details']['seenindicatortype'] = newmessage['details']['seen.indicator_type']
del(newmessage['details']['seen.indicator_type'])
newmessage['summary'] = (
'Bro intel match '
'of {seenindicatortype} '
'in {seenwhere}'
).format(**newmessage['details'])
return (newmessage, metadata)
if logtype == 'known_certs':
if 'serial' not in newmessage['details']:
newmessage['details']['serial'] = '0'
newmessage['summary'] = (
'Certificate X509 seen from: '
'{host}:'
'{port_num}'
).format(**newmessage['details'])
return (newmessage, metadata)
if logtype == 'known_devices':
if 'mac' not in newmessage['details']:
newmessage['details']['mac'] = ''
if 'dhcp_host_name' not in newmessage['details']:
newmessage['details']['dhcp_host_name'] = ''
newmessage['summary'] = (
'New host: '
'{mac}'
).format(**newmessage['details'])
return (newmessage, metadata)
if logtype == 'known_hosts':
if 'host' not in newmessage['details']:
newmessage['details']['host'] = ''
newmessage['summary'] = (
'New host: '
'{host}'
).format(**newmessage['details'])
return (newmessage, metadata)
if logtype == 'known_services':
if 'service' not in newmessage['details']:
newmessage['details']['service'] = []
if not newmessage['details']['service']:
newmessage['details']['service'] = ['Unknown']
if 'host' not in newmessage['details']:
newmessage['details']['host'] = 'unknown'
if 'port_num' not in newmessage['details']:
newmessage['details']['port_num'] = '0'
if 'port_proto' not in newmessage['details']:
newmessage['details']['port_proto'] = ''
newmessage['summary'] = (
'New service: '
'{service[0]} '
'on host '
'{host}:'
'{port_num} / '
'{port_proto}'
).format(**newmessage['details'])
return (newmessage, metadata)
if logtype == 'notice':
newmessage['details']['indicators'] = []
if 'sub' not in newmessage['details']:
newmessage['details']['sub'] = ''
if 'msg' not in newmessage['details']:
newmessage['details']['msg'] = ''
if 'note' not in newmessage['details']:
newmessage['details']['note'] = ''
# clean up the action notice IP addresses
if 'actions' in newmessage['details']:
if newmessage['details']['actions'] == "Notice::ACTION_LOG":
# retrieve indicator ip addresses from the sub field
# "sub": "Indicator: 1.2.3.4, Indicator: 5.6.7.8"
newmessage['details']['indicators'] = [ip for ip in findIPv4(newmessage['details']['sub'])]
# remove the details.src field and add it to indicators
# as it may not be the actual source.
if 'src' in newmessage['details']:
if isIPv4(newmessage['details']['src']):
newmessage['details']['indicators'].append(newmessage['details']['src'])
# If details.src is present overwrite the source IP address with it
newmessage['details']['sourceipaddress'] = newmessage['details']['src']
newmessage['details']['sourceipv4address'] = newmessage['details']['src']
if isIPv6(newmessage['details']['src']):
newmessage['details']['indicators'].append(newmessage['details']['src'])
# If details.src is present overwrite the source IP address with it
newmessage['details']['sourceipv6address'] = newmessage['details']['src']
del newmessage['details']['src']
sumstruct = {}
sumstruct['note'] = newmessage['details']['note']
if 'sourceipv6address' in newmessage['details']:
sumstruct['src'] = newmessage['details']['sourceipv6address']
else:
if 'sourceipv4address' in newmessage['details']:
sumstruct['src'] = newmessage['details']['sourceipv4address']
else:
sumstruct['src'] = 'unknown'
if 'dst' in newmessage['details']:
sumstruct['dst'] = newmessage['details']['dst']
del(newmessage['details']['dst'])
if isIPv4(sumstruct['dst']):
newmessage['details']['destinationipaddress'] = sumstruct['dst']
newmessage['details']['destinationipv4address'] = sumstruct['dst']
if isIPv6(sumstruct['dst']):
newmessage['details']['destinationipv6address'] = sumstruct['dst']
else:
sumstruct['dst'] = 'unknown'
if 'p' in newmessage['details']:
sumstruct['p'] = newmessage['details']['p']
else:
sumstruct['p'] = 'unknown'
newmessage['summary'] = (
'{note} '
'source {src} '
'destination {dst} '
'port {p}'
).format(**sumstruct)
# Thank you for your service
return (newmessage, metadata)
if logtype == 'rdp':
if 'cookie' not in newmessage['details']:
newmessage['details']['cookie'] = 'unknown'
newmessage['summary'] = (
'RDP: {sourceipaddress} -> '
'{destinationipaddress}:'
'{destinationport}'
).format(**newmessage['details'])
return (newmessage, metadata)
if logtype == 'sip':
if 'status_msg' not in newmessage['details']:
newmessage['details']['status_msg'] = 'unknown'
if 'uri' not in newmessage['details']:
newmessage['details']['uri'] = 'unknown'
if 'method' not in newmessage['details']:
newmessage['details']['method'] = 'unknown'
newmessage['summary'] = (
'SIP: {sourceipaddress} -> '
'{destinationipaddress}:'
'{destinationport} '
'method {method} '
'status {status_msg}'
).format(**newmessage['details'])
return (newmessage, metadata)
if logtype == 'software':
newmessage['details']['parsed_version'] = {}
if 'name' not in newmessage['details']:
newmessage['details']['name'] = 'unparsed'
if 'software_type' not in newmessage['details']:
newmessage['details']['software_type'] = 'unknown'
if 'host' not in newmessage['details']:
newmessage['details'] = ''
if 'version.addl' in newmessage['details']:
newmessage['details']['parsed_version']['addl'] = newmessage['details']['version.addl']
del(newmessage['details']['version.addl'])
if 'version.major' in newmessage['details']:
newmessage['details']['parsed_version']['major'] = newmessage['details']['version.major']
del(newmessage['details']['version.major'])
if 'version.minor' in newmessage['details']:
newmessage['details']['parsed_version']['minor'] = newmessage['details']['version.minor']
del(newmessage['details']['version.minor'])
if 'version.minor2' in newmessage['details']:
newmessage['details']['parsed_version']['minor2'] = newmessage['details']['version.minor2']
del(newmessage['details']['version.minor2'])
if 'version.minor3' in newmessage['details']:
newmessage['details']['parsed_version']['minor3'] = newmessage['details']['version.minor3']
del(newmessage['details']['version.minor3'])
newmessage['summary'] = (
'Found {software_type} software '
'on {host}'
).format(**newmessage['details'])
return (newmessage, metadata)
if logtype == 'socks':
if 'version' not in newmessage['details']:
newmessage['details']['version'] = '0'
if 'status' not in newmessage['details']:
newmessage['details']['status'] = 'unknown'
newmessage['summary'] = (
'SOCKSv{version}: '
'{sourceipaddress} -> '
'{destinationipaddress}:'
'{destinationport} '
'status {status}'
).format(**newmessage['details'])
return (newmessage, metadata)
if logtype == 'dce_rpc':
if 'endpoint' not in newmessage['details']:
newmessage['details']['endpoint'] = 'unknown'
if 'operation' not in newmessage['details']:
newmessage['details']['operation'] = 'unknown'
newmessage['summary'] = (
'DCERPC: {sourceipaddress} -> '
'{destinationipaddress}:'
'{destinationport}'
).format(**newmessage['details'])
return (newmessage, metadata)
if logtype == 'kerberos':
if 'request_type' not in newmessage['details']:
newmessage['details']['request_type'] = 'unknown'
if 'client' not in newmessage['details']:
newmessage['details']['client'] = 'unknown'
if 'service' not in newmessage['details']:
newmessage['details']['service'] = 'unknown'
if 'error_msg' not in newmessage['details']:
newmessage['details']['error_msg'] = ''
newmessage['summary'] = (
'{sourceipaddress} -> '
'{destinationipaddress}:'
'{destinationport} '
'request {request_type}'
).format(**newmessage['details'])
if 'success' in newmessage['details']:
newmessage['summary'] += ' success {0}'.format(newmessage['details']['success'])
else:
newmessage['summary'] += ' success unknown'
return (newmessage, metadata)
if logtype == 'ntlm':
newmessage['details']['ntlm'] = {}
if 'domainname' in newmessage['details']:
newmessage['details']['ntlm']['domainname'] = newmessage['details']['domainname']
del(newmessage['details']['domainname'])
else:
newmessage['details']['ntlm']['domainname'] = 'unknown'
if 'hostname' in newmessage['details']:
newmessage['details']['ntlm']['hostname'] = newmessage['details']['hostname']
del(newmessage['details']['hostname'])
else:
newmessage['details']['ntlm']['hostname'] = 'unknown'
if 'username' in newmessage['details']:
newmessage['details']['ntlm']['username'] = newmessage['details']['username']
del(newmessage['details']['username'])
else:
newmessage['details']['ntlm']['username'] = '******'
if 'status' not in newmessage['details']:
newmessage['details']['status'] = 'unknown'
newmessage['summary'] = (
'NTLM: {sourceipaddress} -> '
'{destinationipaddress}:'
'{destinationport} '
).format(**newmessage['details'])
if 'success' in newmessage['details']:
newmessage['summary'] += 'success {0} status {1}'.format(newmessage['details']['success'],newmessage['details']['status'])
else:
newmessage['summary'] += 'success unknown status {0}'.format(newmessage['details']['status'])
return (newmessage, metadata)
if logtype == 'smb_files':
newmessage['details']['smbtimes'] = {}
if 'path' not in newmessage['details']:
newmessage['details']['path'] = ''
if 'name' not in newmessage['details']:
newmessage['details']['name'] = ''
if 'action' not in newmessage['details']:
newmessage['details']['action'] = ''
if 'times.accessed' in newmessage['details']:
newmessage['details']['smbtimes']['accessed'] = toUTC(float(newmessage['details']['times.accessed'])).isoformat()
del(newmessage['details']['times.accessed'])
if 'times.changed' in newmessage['details']:
newmessage['details']['smbtimes']['changed'] = toUTC(float(newmessage['details']['times.changed'])).isoformat()
del(newmessage['details']['times.changed'])
if 'times.created' in newmessage['details']:
newmessage['details']['smbtimes']['created'] = toUTC(float(newmessage['details']['times.created'])).isoformat()
del(newmessage['details']['times.created'])
if 'times.modified' in newmessage['details']:
newmessage['details']['smbtimes']['modified'] = toUTC(float(newmessage['details']['times.modified'])).isoformat()
del(newmessage['details']['times.modified'])
newmessage['summary'] = (
'SMB file: '
'{sourceipaddress} -> '
'{destinationipaddress}:'
'{destinationport} '
'{action}'
).format(**newmessage['details'])
return(newmessage, metadata)
if logtype == 'smb_mapping':
if 'share_type' not in newmessage['details']:
newmessage['details']['share_type'] = ''
if 'path' not in newmessage['details']:
newmessage['details']['path'] = ''
newmessage['summary'] = (
'SMB mapping: '
'{sourceipaddress} -> '
'{destinationipaddress}:'
'{destinationport} '
'{share_type}'
).format(**newmessage['details'])
return(newmessage, metadata)
if logtype == 'snmp':
if 'version' not in newmessage['details']:
newmessage['details']['version'] = 'Unknown'
if 'get_bulk_requests' not in newmessage['details']:
newmessage['details']['get_bulk_requests'] = 0
if 'get_requests' not in newmessage['details']:
newmessage['details']['get_requests'] = 0
if 'set_requests' not in newmessage['details']:
newmessage['details']['set_requests'] = 0
if 'get_responses' not in newmessage['details']:
newmessage['details']['get_responses'] = 0
newmessage['details']['getreqestssum'] = '{0}'.format(newmessage['details']['get_bulk_requests'] + newmessage['details']['get_requests'])
newmessage['summary'] = (
'SNMPv{version}: '
'{sourceipaddress} -> '
'{destinationipaddress}:'
'{destinationport} '
'({getreqestssum} get / '
'{set_requests} set requests '
'{get_responses} get responses)'
).format(**newmessage['details'])
return (newmessage, metadata)
if logtype == 'x509':
newmessage['details']['certificate'] = {}
if 'basic_constraints.ca' in newmessage['details']:
newmessage['details']['certificate']['basic_constraints_ca'] = newmessage['details']['basic_constraints.ca']
del(newmessage['details']['basic_constraints.ca'])
if 'basic_constraints.path_len' in newmessage['details']:
newmessage['details']['certificate']['basic_constraints_path_len'] = newmessage['details']['basic_constraints.path_len']
del(newmessage['details']['basic_constraints.path_len'])
if 'certificate.exponent' in newmessage['details']:
newmessage['details']['certificate']['exponent'] = newmessage['details']['certificate.exponent']
del(newmessage['details']['certificate.exponent'])
if 'certificate.issuer' in newmessage['details']:
newmessage['details']['certificate']['issuer'] = newmessage['details']['certificate.issuer']
del(newmessage['details']['certificate.issuer'])
if 'certificate.key_alg' in newmessage['details']:
newmessage['details']['certificate']['key_alg'] = newmessage['details']['certificate.key_alg']
del(newmessage['details']['certificate.key_alg'])
if 'certificate.key_length' in newmessage['details']:
newmessage['details']['certificate']['key_length'] = newmessage['details']['certificate.key_length']
del(newmessage['details']['certificate.key_length'])
if 'certificate.key_type' in newmessage['details']:
newmessage['details']['certificate']['key_type'] = newmessage['details']['certificate.key_type']
del(newmessage['details']['certificate.key_type'])
if 'certificate.not_valid_after' in newmessage['details']:
newmessage['details']['certificate']['not_valid_after'] = toUTC(float(newmessage['details']['certificate.not_valid_after'])).isoformat()
del(newmessage['details']['certificate.not_valid_after'])
if 'certificate.not_valid_before' in newmessage['details']:
newmessage['details']['certificate']['not_valid_before'] = toUTC(float(newmessage['details']['certificate.not_valid_before'])).isoformat()
del(newmessage['details']['certificate.not_valid_before'])
if 'certificate.sig_alg' in newmessage['details']:
newmessage['details']['certificate']['sig_alg'] = newmessage['details']['certificate.sig_alg']
del(newmessage['details']['certificate.sig_alg'])
if 'certificate.subject' in newmessage['details']:
newmessage['details']['certificate']['subject'] = newmessage['details']['certificate.subject']
del(newmessage['details']['certificate.subject'])
if 'certificate.version' in newmessage['details']:
newmessage['details']['certificate']['version'] = newmessage['details']['certificate.version']
del(newmessage['details']['certificate.version'])
if 'certificate.serial' in newmessage['details']:
newmessage['details']['certificate']['serial'] = newmessage['details']['certificate.serial']
del(newmessage['details']['certificate.serial'])
else:
newmessage['details']['certificate']['serial'] = '0'
newmessage['summary'] = (
'X509 certificate seen'
).format(**newmessage['details']['certificate'])
return (newmessage, metadata)
return (newmessage, metadata)
def test_basic_dict_negative(self):
assert key_exists('otherkey', {'details': 'abcd'}) is False
def test_basic_dict_positive_complicated_2(self):
assert key_exists('details.depth2.depth1', self.default_event) is False
