def onMessage(self, message, metadata): # check for category like 'ldap' and rename the tls field if key_exists('category', message): data = message.get('category') if data == 'ldap': if key_exists('details.tls', message): message['details']['tls_encrypted'] = message['details'][ 'tls'] del (message['details']['tls']) if 'source' not in message: message['source'] = 'ldap' details = message.get('details', {}) actor_str = details.get('actor', '') actor_email = _parse_email_from_actor(actor_str) if actor_email is None: details['email'] = None details['username'] = None message['details'] = details return (message, metadata) username = actor_email.split('@')[0] details['email'] = actor_email details['username'] = username message['details'] = details return (message, metadata)
def onMessage(self, message, metadata): if 'source' not in message: return (message, metadata) if not message['source'] == 'guardduty': return (message, metadata) # reformat the date fields to iosformat for date_key in self.date_keys: if key_exists(date_key, message): message = self.convert_key_date_format(date_key, message) # convert the dict to a dot dict for saner deep key/value processing message = DotDict(message) # pull out the likely source IP address for ipaddress_key in self.ipaddress_keys: if 'sourceipaddress' not in message['details'].keys(): if key_exists(ipaddress_key, message): message.details.sourceipaddress = message.get( ipaddress_key) # if we still haven't found what we are looking for #U2 # sometimes it's in a list if 'sourceipaddress' not in message['details'].keys(): if key_exists('details.finding.action.portProbeAction.portProbeDetails', message) \ and isinstance(message.details.finding.action.portProbeAction.portProbeDetails, list): # inspect the first list entry and see if it contains an IP portProbeDetails = DotDict(message.details.finding.action. portProbeAction.portProbeDetails[0]) if key_exists('remoteIpDetails.ipAddressV4', portProbeDetails): message.details.sourceipaddress = portProbeDetails.remoteIpDetails.ipAddressV4 # recovert the message back to a plain dict return (dict(message), metadata)
def onMessage(self, message, metadata): # check for category like 'ldap' and rename the tls field if key_exists('category', message): data = message.get('category') if data == 'ldap': if key_exists('details.tls', message): message['details']['tls_encrypted'] = message['details'][ 'tls'] del (message['details']['tls']) if 'source' not in message: message['source'] = 'ldap' return (message, metadata)
def on_message(self, message_raw): if "Message" in message_raw: message = json.loads(message_raw["Message"]) if key_exists('details.finding.action.actionType', message): if message["details"]["finding"]["action"]["actionType"] == "PORT_PROBE": if "portProbeDetails" in message["details"]["finding"]["action"]["portProbeAction"]: for probe in message["details"]["finding"]["action"]["portProbeAction"]["portProbeDetails"]: isolatedmessage = message isolatedmessage["details"]["finding"]["probeevent"] = probe self.build_submit_message(isolatedmessage) elif message["details"]["finding"]["action"]["actionType"] == "AWS_API_CALL": # Fixup the api call data somewhat, depending on what we were given if "recentApiCalls" in message["details"]["finding"]["additionalInfo"]: message["details"]["finding"]["additionalInfo"]["apiCalls"] = message["details"]["finding"][ "additionalInfo" ]["recentApiCalls"] if "apiCalls" not in message["details"]["finding"]["additionalInfo"]: message["details"]["finding"]["additionalInfo"]["apiCalls"] = message["details"]["finding"][ "action" ]["awsApiCallAction"] if type(message["details"]["finding"]["additionalInfo"]["apiCalls"]) == list: for call in message["details"]["finding"]["additionalInfo"]["apiCalls"]: isolatedmessage = message isolatedmessage["details"]["finding"]["apicalls"] = call self.build_submit_message(isolatedmessage) else: isolatedmessage = message isolatedmessage["details"]["finding"]["apicalls"] = message["details"]["finding"][ "additionalInfo" ]["apiCalls"] self.build_submit_message(isolatedmessage) else: self.build_submit_message(message)
def onMessage(self, message, metadata): ''' Check if source is in the message, if not then add cloudtrail as the source. ''' if 'source' not in message: return (message, metadata) if not message['source'] == 'cloudtrail': return (message, metadata) ''' Check if details.requestparameters.htmlpart exists, if it does it's generally longer than 32000 bytes, so we'll truncate it to 4096 characters using the constant ES_FIELD_VALUE_LIMIT so that ES will ingest it, leaving us with knowledge of what the field contains without the overkill of storing the entire page. ''' ES_FIELD_VALUE_LIMIT = 4095 if 'requestparameters' in message['details'] and message['details']['requestparameters']: if 'htmlpart' in message['details']['requestparameters'] and message['details']['requestparameters']['htmlpart']: message['details']['requestparameters']['htmlpart'] = message['details']['requestparameters']['htmlpart'][0:ES_FIELD_VALUE_LIMIT] for modified_key in self.modify_keys: if key_exists(modified_key, message): message = self.convert_key_raw_str(modified_key, message) return (message, metadata)
def on_message(self, message_raw): if "Message" in message_raw: message = json.loads(message_raw["Message"]) if key_exists('details.finding.action.actionType', message): if message["details"]["finding"]["action"][ "actionType"] == "PORT_PROBE": if "portProbeDetails" in message["details"]["finding"][ "action"]["portProbeAction"]: for probe in message["details"]["finding"]["action"][ "portProbeAction"]["portProbeDetails"]: isolatedmessage = message isolatedmessage["details"]["finding"][ "probeevent"] = probe self.build_submit_message(isolatedmessage) elif message["details"]["finding"]["action"][ "actionType"] == "AWS_API_CALL": if "recentApiCalls" in message["details"]["finding"][ "additionalInfo"]: message["details"]["finding"]["additionalInfo"][ "apiCalls"] = message["details"]["finding"][ "additionalInfo"]["recentApiCalls"] for call in message["details"]["finding"][ "additionalInfo"]["apiCalls"]: isolatedmessage = message isolatedmessage["details"]["finding"][ "apicalls"] = call self.build_submit_message(isolatedmessage) else: self.build_submit_message(message)
def onMessage(self, message, metadata): if key_exists('tags', message): if 'zoom' not in message['tags']: return (message, metadata) newmessage = {} newmessage['details'] = {} newmessage['category'] = 'zoom' newmessage['eventsource'] = 'MozDef-EF-zoom' newmessage['source'] = 'api_aws_lambda' newmessage['hostname'] = 'zoom_host' newmessage['severity'] = 'info' newmessage['processname'] = 'zoom_webhook_api' if key_exists('tags', message): newmessage['tags'] = message['tags'] if key_exists('details.event', message): newmessage['details']['event'] = message['details']['event'] else: newmessage['details']['event'] = 'UNKNOWN' if key_exists('details.payload.account_id', message): newmessage['details']['account_id'] = message['details']['payload']['account_id'] if key_exists('details.payload.object.account_id', message): if newmessage.get('details.account_id') != message.get('details.payload.object.account_id'): newmessage['details']['meeting_account_id'] = message['details']['payload']['object']['account_id'] elif key_exists('details.payload.object.account_id', message): newmessage['details']['account_id'] = message['details']['payload']['object']['account_id'] # rewrite summary to be more informative newmessage['summary'] = "" if key_exists('details.event', newmessage): newmessage['summary'] = "zoom: {0}".format(message['details']['event']) if key_exists('details.payload.object.participant.user_name', message): newmessage['summary'] += " triggered by user {0}".format(message['details']['payload']['object']['participant']['user_name']) elif key_exists('details.payload.operator', message): newmessage['summary'] += " triggered by user {0}".format(message['details']['payload']['operator']) # iterate through top level keys - push, etc if newmessage['source'] in self.eventtypes: for key in self.yap[newmessage['source']]: mappedvalue = jmespath.search(self.yap[newmessage['source']][key], message) # JMESPath likes to silently return a None object if mappedvalue is not None: newmessage['details'][key] = mappedvalue else: newmessage = None return (newmessage, metadata)
def onMessage(self, message, metadata): if 'source' not in message: return (message, metadata) if not message['source'] == 'cloudtrail': return (message, metadata) for modified_key in self.modify_keys: if key_exists(modified_key, message): message = self.convert_key_raw_str(modified_key, message) return (message, metadata)
def onMessage(self, message, metadata): # check for messages we have vetted as n/a and prevalent # from a sec standpoint and drop them also rewrite fields # to drop unecessary expansion # omit "topic" field if key_exists('details.payload.object.topic', message): del message['details']['payload']['object']['topic'] # rewrite summary to be more informative message['summary'] = "" if key_exists('details.event', message): message['summary'] = "zoom: {0}".format( message['details']['event']) if key_exists('details.payload.object.participant.user_name', message): message['summary'] += " triggered by user {0}".format( message['details']['payload']['object']['participant'] ['user_name']) elif key_exists('details.payload.operator', message): message['summary'] += " triggered by user {0}".format( message['details']['payload']['operator']) # drop duplicated account_id field if key_exists('details.payload.account_id', message) and key_exists( 'details.payload.object.account_id', message): if message.get('details.payload.account_id') == message.get( 'details.payload.object.account_id'): del message['details']['payload']['object']['account_id'] return (message, metadata)
def onMessage(self, message, metadata): if 'source' not in message: return (message, metadata) if not message['source'] == 'guardduty': return (message, metadata) # reformat the date fields to iosformat for date_key in self.date_keys: if key_exists(date_key, message): if message.get(date_key) is None: continue else: message = self.convert_key_date_format(date_key, message) # convert the dict to a dot dict for saner deep key/value processing message = DotDict(message) # pull out the likely source IP address for ipaddress_key in self.ipaddress_keys: if 'sourceipaddress' not in message['details']: if key_exists(ipaddress_key, message): message.details.sourceipaddress = message.get( ipaddress_key) # if we still haven't found what we are looking for #U2 # sometimes it's in a list if 'sourceipaddress' not in message['details']: if key_exists('details.finding.action.portprobeaction.portprobedetails', message) \ and isinstance(message.details.finding.action.portprobeaction.portprobedetails, list): # inspect the first list entry and see if it contains an IP portprobedetails = DotDict( message.details.finding.action.portprobeaction.portprobedetails[0]) if key_exists('remoteipdetails.ipaddressv4', portprobedetails): message.details.sourceipaddress = portprobedetails.remoteipdetails.ipaddressv4 # recovert the message back to a plain dict return (dict(message), metadata)
def test_basic_dict_positive_complicated_2(self): assert key_exists('details.depth2.depth1', self.default_event) is False
def test_basic_dict_positive_complicated(self): assert key_exists('details.depth1.depth2.depth3.somekey', self.default_event) is True
def test_basic_dict_negative(self): assert key_exists('otherkey', {'details': 'abcd'}) is False
def test_basic_dict_positive(self): assert key_exists('details', {'details': 'abcd'}) is True
def onMessage(self, message, metadata): if key_exists('tags', message): if 'zoom' not in message['tags']: return (message, metadata) newmessage = {} newmessage['details'] = {} newmessage['category'] = 'zoom' newmessage['eventsource'] = 'MozDef-EF-zoom' newmessage['source'] = 'api_aws_lambda' newmessage['hostname'] = 'zoom_host' newmessage['severity'] = 'info' newmessage['processname'] = 'zoom_webhook_api' if key_exists('tags', message): newmessage['tags'] = message['tags'] if key_exists('details.event', message): newmessage['details']['event'] = message['details']['event'] else: newmessage['details']['event'] = 'UNKNOWN' if key_exists('details.payload.account_id', message): newmessage['details']['account_id'] = message['details'][ 'payload']['account_id'] if key_exists('details.payload.object.account_id', message): if newmessage.get('details.account_id') != message.get( 'details.payload.object.account_id'): newmessage['details']['meeting_account_id'] = message[ 'details']['payload']['object']['account_id'] elif key_exists('details.payload.object.account_id', message): newmessage['details']['account_id'] = message['details'][ 'payload']['object']['account_id'] # rewrite summary to be more informative newmessage['summary'] = "" if key_exists('details.event', newmessage): newmessage['summary'] = "zoom: {0}".format( message['details']['event']) if key_exists('details.payload.object.participant.user_name', message): newmessage['summary'] += " triggered by user {0}".format( message['details']['payload']['object']['participant'] ['user_name']) elif key_exists('details.payload.operator', message): newmessage['summary'] += " triggered by user {0}".format( message['details']['payload']['operator']) # iterate through top level keys - push, etc if newmessage['source'] in self.eventtypes: for key in self.yap[newmessage['source']]: mappedvalue = jmespath.search( self.yap[newmessage['source']][key], message) # JMESPath likes to silently return a None object if mappedvalue is not None: newmessage['details'][key] = mappedvalue # Some zoom messages don't contain values within details.payload.object.start_time or details.payload.old_object.start_time. # so we set it to original start time, if there is no time data, we remove the key from the message. if key_exists('details.payload.object.start_time', message): if message['details']['payload']['object']['start_time'] != '': newmessage['details']['start_time'] = message['details'][ 'payload']['object']['start_time'] else: del newmessage['details']['start_time'] if key_exists('details.payload.old_object.start_time', message): if message['details']['payload']['old_object'][ 'start_time'] != '': newmessage['details'][ 'original_sched_start_time'] = message['details'][ 'payload']['old_object']['start_time'] else: del newmessage['details']['original_sched_start_time'] # Duration can exist in details.payload.object and details.payload.old_object, let's ensure we are capturing these correctly for updated meetings. if key_exists('details.payload.object.duration', message): newmessage['details']['duration'] = message['details'][ 'payload']['object']['duration'] if key_exists('details.payload.old_object.duration', message): newmessage['details']['original_sched_duration'] = message[ 'details']['payload']['old_object']['duration'] else: newmessage = None return (newmessage, metadata)
def onMessage(self, message, metadata): if 'tags' not in message: return (message, metadata) if 'githubeventsqs' not in message['tags']: return (message, metadata) newmessage = {} newmessage['details'] = {} newmessage['category'] = 'github' newmessage['tags'] = ['github', 'webhook'] newmessage['eventsource'] = 'githubeventsqs' if key_exists('details.event', message): newmessage['source'] = message['details']['event'] else: newmessage['source'] = 'UNKNOWN' if key_exists('details.request_id', message): newmessage['details']['request_id'] = message['details']['request_id'] else: newmessage['details']['request_id'] = 'UNKNOWN' # iterate through top level keys - push, etc if newmessage['source'] in self.eventtypes: for key in self.yap[newmessage['source']]: mappedvalue = jmespath.search(self.yap[newmessage['source']][key], message) # JMESPath likes to silently return a None object if mappedvalue is not None: newmessage['details'][key] = mappedvalue if 'commit_ts' in newmessage['details']: newmessage['timestamp'] = newmessage['details']['commit_ts'] newmessage['utctimestamp'] = toUTC(newmessage['details']['commit_ts']).isoformat() else: newmessage = None if key_exists('source', newmessage) and newmessage.get('source') is not 'UNKNOWN': newmessage['summary'] = "github: {0}: ".format(newmessage['source']) if key_exists('source', newmessage) and newmessage.get('source') is 'installation': newmessage['summary'] = "github app: {0} ".format(newmessage['source']) if key_exists('source', newmessage) and newmessage.get('source') is 'public': newmessage['summary'] = "github : change from private to {0} ".format(newmessage['source']) if key_exists('details.status', newmessage): action_status = "{0} ".format(newmessage['details']['status']) newmessage['summary'] += action_status if key_exists('details.action', newmessage): github_action = "{0} ".format(newmessage['details']['action']) newmessage['summary'] += github_action if key_exists('details.ref_type', newmessage): reference = "{0} ".format(newmessage['details']['ref_type']) newmessage['summary'] += reference if key_exists('details.repo_name', newmessage): repository_name = "on repo: {0} ".format(newmessage['details']['repo_name']) newmessage['summary'] += repository_name if key_exists('details.alert_note', newmessage): sec_advisory = "for: {0}".format(newmessage['details']['alert_note']) newmessage['summary'] += sec_advisory if key_exists('details.alert_package', newmessage): vuln_package = "package: {0} ".format(newmessage['details']['alert_package']) newmessage['summary'] += vuln_package if key_exists('details.team_name', newmessage): team_name = "team: {0} ".format(newmessage['details']['team_name']) newmessage['summary'] += team_name if key_exists('details.blocked_user_login', newmessage): blocked_user = "******".format(newmessage['details']['blocked_user_login']) newmessage['summary'] += blocked_user if key_exists('details.org_login', newmessage): org_name = "in org: {0} ".format(newmessage['details']['org_login']) newmessage['summary'] += org_name if key_exists('details.username', newmessage): github_user = "******".format(newmessage['details']['username']) newmessage['summary'] += "triggered by " + github_user return (newmessage, metadata)
def onMessage(self, message, metadata): # make sure I really wanted to see this message # bail out early if not if 'customendpoint' not in message: return message, metadata if 'category' not in message: return message, metadata if 'SOURCE' not in message: return message, metadata if message['category'] != 'bro': return message, metadata # move Bro specific fields under 'details' while preserving metadata newmessage = dict() # default replacement for old _type subcategory. # to preserve filtering capabilities newmessage['type']= 'nsm' try: newmessage['details'] = json.loads(message['MESSAGE']) except: newmessage['details'] = {} newmessage['rawdetails'] = message newmessage['customendpoint'] = 'bro' # move some fields that are expected at the event 'root' where they belong if 'HOST_FROM' in message: newmessage['hostname'] = message['HOST_FROM'] if 'tags' in message: newmessage['tags'] = message['tags'] if 'category' in message: newmessage['category'] = message['category'] if 'SOURCE' in message: # transform bro_files into files fast newmessage['source'] = message['SOURCE'][4:] if 'resp_cc' in newmessage['details']: del(newmessage['details']['resp_cc']) # add mandatory fields if 'ts' in newmessage['details']: newmessage['utctimestamp'] = toUTC(float(newmessage['details']['ts'])).isoformat() newmessage['timestamp'] = toUTC(float(newmessage['details']['ts'])).isoformat() # del(newmessage['details']['ts']) else: # a malformed message somehow managed to crawl to us, let's put it somewhat together newmessage['utctimestamp'] = toUTC(datetime.now()).isoformat() newmessage['timestamp'] = toUTC(datetime.now()).isoformat() newmessage['receivedtimestamp'] = toUTC(datetime.now()).isoformat() newmessage['eventsource'] = 'nsm' newmessage['severity'] = 'INFO' newmessage['mozdefhostname'] = self.mozdefhostname if 'id.orig_h' in newmessage['details']: newmessage['details']['sourceipaddress'] = newmessage['details']['id.orig_h'] del(newmessage['details']['id.orig_h']) if 'id.orig_p' in newmessage['details']: newmessage['details']['sourceport'] = newmessage['details']['id.orig_p'] del(newmessage['details']['id.orig_p']) if 'id.resp_h' in newmessage['details']: newmessage['details']['destinationipaddress'] = newmessage['details']['id.resp_h'] del(newmessage['details']['id.resp_h']) if 'id.resp_p' in newmessage['details']: newmessage['details']['destinationport'] = newmessage['details']['id.resp_p'] del(newmessage['details']['id.resp_p']) if 'details' in newmessage: if 'FILE_NAME' in newmessage['details']: del(newmessage['details']['FILE_NAME']) if 'MESSAGE' in newmessage['details']: del(newmessage['details']['MESSAGE']) if 'SOURCE' in newmessage['details']: del(newmessage['details']['SOURCE']) # All Bro logs need special treatment, so we provide it # Not a known log source? Mark it as such and return if 'source' not in newmessage: newmessage['source'] = 'unknown' return newmessage, metadata else: logtype = newmessage['source'] if logtype == 'conn': newmessage['details']['originipbytes'] = newmessage['details']['orig_ip_bytes'] newmessage['details']['responseipbytes'] = newmessage['details']['resp_ip_bytes'] del(newmessage['details']['orig_ip_bytes']) del(newmessage['details']['resp_ip_bytes']) if 'history' not in newmessage['details']: newmessage['details']['history'] = '' newmessage['summary'] = ( '{sourceipaddress}:'+ '{sourceport} -> '+ '{destinationipaddress}:' '{destinationport} '+ '{history} '+ '{originipbytes} bytes / ' '{responseipbytes} bytes' ).format(**newmessage['details']) return (newmessage, metadata) if logtype == 'files': if 'rx_hosts' in newmessage['details']: newmessage['details']['sourceipaddress'] = '{0}'.format(newmessage['details']['rx_hosts'][0]) if 'tx_hosts' in newmessage['details']: newmessage['details']['destinationipaddress'] = '{0}'.format(newmessage['details']['tx_hosts'][0]) if 'mime_type' not in newmessage['details']: newmessage['details']['mime_type'] = 'unknown' if 'filename' not in newmessage['details']: newmessage['details']['filename'] = 'unknown' if 'total_bytes' not in newmessage['details']: newmessage['details']['total_bytes'] = '0' if 'md5' not in newmessage['details']: newmessage['details']['md5'] = 'None' if 'filesource' not in newmessage['details']: newmessage['details']['filesource'] = 'None' newmessage['summary'] = ( '{rx_hosts[0]} ' 'downloaded (MD5) ' '{md5} ' 'MIME {mime_type} ' '({total_bytes} bytes) ' 'from {tx_hosts[0]} ' 'via {filesource}' ).format(**newmessage['details']) return (newmessage, metadata) if logtype == 'dns': if 'qtype_name' not in newmessage['details']: newmessage['details']['qtype_name'] = 'unknown' if 'query' not in newmessage['details']: newmessage['details']['query'] = '' if 'rcode_name' not in newmessage['details']: newmessage['details']['rcode_name'] = '' newmessage['summary'] = ( 'DNS {qtype_name} type query ' '{sourceipaddress} -> ' '{destinationipaddress}:{destinationport}' ).format(**newmessage['details']) return (newmessage, metadata) if logtype == 'http': if 'method' not in newmessage['details']: newmessage['details']['method'] = '' if 'host' not in newmessage['details']: newmessage['details']['host'] = '' if 'uri' not in newmessage['details']: newmessage['details']['uri'] = '' newmessage['details']['uri'] = newmessage['details']['uri'][:1024] if 'status_code' not in newmessage['details']: newmessage['details']['status_code'] = '' newmessage['summary'] = ( 'HTTP {method} ' '{sourceipaddress} -> ' '{destinationipaddress}:' '{destinationport}' ).format(**newmessage['details']) return (newmessage, metadata) if logtype == 'ssl': if 'server_name' not in newmessage['details']: # fake it till you make it newmessage['details']['server_name'] = newmessage['details']['destinationipaddress'] newmessage['summary'] = ( 'SSL: {sourceipaddress} -> ' '{destinationipaddress}:' '{destinationport}' ).format(**newmessage['details']) return (newmessage, metadata) if logtype == 'dhcp': if 'assigned_addr' not in newmessage['details']: newmessage['details']['assigned_addr'] = "0.0.0.0" if 'mac' not in newmessage['details']: newmessage['details']['mac'] = "000000000000" newmessage['details']['mac'] = newmessage['details']['mac'].replace(':', '') newmessage['summary'] = ( '{assigned_addr} assigned to ' '{mac}' ).format(**newmessage['details']) return (newmessage, metadata) if logtype == 'ftp': if 'command' not in newmessage['details']: newmessage['details']['command'] = '' if 'user' not in newmessage['details']: newmessage['details']['user'] = '' newmessage['summary'] = ( 'FTP: {sourceipaddress} -> ' '{destinationipaddress}:' '{destinationport}' ).format(**newmessage['details']) return (newmessage, metadata) if logtype == 'pe': if 'os' not in newmessage['details']: newmessage['details']['os'] = '' if 'subsystem' not in newmessage['details']: newmessage['details']['subsystem'] = '' newmessage['summary'] = ( 'PE file: {os} ' '{subsystem}' ).format(**newmessage['details']) return (newmessage, metadata) if logtype == 'smtp': newmessage['summary'] = ( 'SMTP: {sourceipaddress} -> ' '{destinationipaddress}:' '{destinationport}' ).format(**newmessage['details']) if key_exists('details.tls', newmessage): newmessage['details']['tls_encrypted'] = newmessage['details']['tls'] del(newmessage['details']['tls']) return (newmessage, metadata) if logtype == 'ssh': summary = ( 'SSH: {sourceipaddress} -> ' '{destinationipaddress}:' '{destinationport}' ).format(**newmessage['details']) if 'auth_success' in newmessage['details']: summary += ' success {0}'.format(newmessage['details']['auth_success']) newmessage['summary'] = summary return (newmessage, metadata) if logtype == 'tunnel': if 'tunnel_type' not in newmessage['details']: newmessage['details']['tunnel_type'] = '' if 'action' not in newmessage['details']: newmessage['details']['action'] = '' newmessage['summary'] = ( '{sourceipaddress} -> ' '{destinationipaddress}:' '{destinationport} ' '{tunnel_type} ' '{action}' ).format(**newmessage['details']) return (newmessage, metadata) if logtype == 'intel': if 'seen.indicator' in newmessage['details']: newmessage['details']['seenindicator'] = newmessage['details']['seen.indicator'] del(newmessage['details']['seen.indicator']) else: newmessage['details']['seenindicator'] = '' if 'seen.node' in newmessage['details']: newmessage['details']['seennode'] = newmessage['details']['seen.node'] del(newmessage['details']['seen.node']) if 'seen.where' in newmessage['details']: newmessage['details']['seenwhere'] = newmessage['details']['seen.where'] del(newmessage['details']['seen.where']) if 'seen.indicator_type' in newmessage['details']: newmessage['details']['seenindicatortype'] = newmessage['details']['seen.indicator_type'] del(newmessage['details']['seen.indicator_type']) newmessage['summary'] = ( 'Bro intel match ' 'of {seenindicatortype} ' 'in {seenwhere}' ).format(**newmessage['details']) return (newmessage, metadata) if logtype == 'known_certs': if 'serial' not in newmessage['details']: newmessage['details']['serial'] = '0' newmessage['summary'] = ( 'Certificate X509 seen from: ' '{host}:' '{port_num}' ).format(**newmessage['details']) return (newmessage, metadata) if logtype == 'known_devices': if 'mac' not in newmessage['details']: newmessage['details']['mac'] = '' if 'dhcp_host_name' not in newmessage['details']: newmessage['details']['dhcp_host_name'] = '' newmessage['summary'] = ( 'New host: ' '{mac}' ).format(**newmessage['details']) return (newmessage, metadata) if logtype == 'known_hosts': if 'host' not in newmessage['details']: newmessage['details']['host'] = '' newmessage['summary'] = ( 'New host: ' '{host}' ).format(**newmessage['details']) return (newmessage, metadata) if logtype == 'known_services': if 'service' not in newmessage['details']: newmessage['details']['service'] = [] if not newmessage['details']['service']: newmessage['details']['service'] = ['Unknown'] if 'host' not in newmessage['details']: newmessage['details']['host'] = 'unknown' if 'port_num' not in newmessage['details']: newmessage['details']['port_num'] = '0' if 'port_proto' not in newmessage['details']: newmessage['details']['port_proto'] = '' newmessage['summary'] = ( 'New service: ' '{service[0]} ' 'on host ' '{host}:' '{port_num} / ' '{port_proto}' ).format(**newmessage['details']) return (newmessage, metadata) if logtype == 'notice': newmessage['details']['indicators'] = [] if 'sub' not in newmessage['details']: newmessage['details']['sub'] = '' if 'msg' not in newmessage['details']: newmessage['details']['msg'] = '' if 'note' not in newmessage['details']: newmessage['details']['note'] = '' # clean up the action notice IP addresses if 'actions' in newmessage['details']: if newmessage['details']['actions'] == "Notice::ACTION_LOG": # retrieve indicator ip addresses from the sub field # "sub": "Indicator: 1.2.3.4, Indicator: 5.6.7.8" newmessage['details']['indicators'] = [ip for ip in findIPv4(newmessage['details']['sub'])] # remove the details.src field and add it to indicators # as it may not be the actual source. if 'src' in newmessage['details']: if isIPv4(newmessage['details']['src']): newmessage['details']['indicators'].append(newmessage['details']['src']) # If details.src is present overwrite the source IP address with it newmessage['details']['sourceipaddress'] = newmessage['details']['src'] newmessage['details']['sourceipv4address'] = newmessage['details']['src'] if isIPv6(newmessage['details']['src']): newmessage['details']['indicators'].append(newmessage['details']['src']) # If details.src is present overwrite the source IP address with it newmessage['details']['sourceipv6address'] = newmessage['details']['src'] del newmessage['details']['src'] sumstruct = {} sumstruct['note'] = newmessage['details']['note'] if 'sourceipv6address' in newmessage['details']: sumstruct['src'] = newmessage['details']['sourceipv6address'] else: if 'sourceipv4address' in newmessage['details']: sumstruct['src'] = newmessage['details']['sourceipv4address'] else: sumstruct['src'] = 'unknown' if 'dst' in newmessage['details']: sumstruct['dst'] = newmessage['details']['dst'] del(newmessage['details']['dst']) if isIPv4(sumstruct['dst']): newmessage['details']['destinationipaddress'] = sumstruct['dst'] newmessage['details']['destinationipv4address'] = sumstruct['dst'] if isIPv6(sumstruct['dst']): newmessage['details']['destinationipv6address'] = sumstruct['dst'] else: sumstruct['dst'] = 'unknown' if 'p' in newmessage['details']: sumstruct['p'] = newmessage['details']['p'] else: sumstruct['p'] = 'unknown' newmessage['summary'] = ( '{note} ' 'source {src} ' 'destination {dst} ' 'port {p}' ).format(**sumstruct) # Thank you for your service return (newmessage, metadata) if logtype == 'rdp': if 'cookie' not in newmessage['details']: newmessage['details']['cookie'] = 'unknown' newmessage['summary'] = ( 'RDP: {sourceipaddress} -> ' '{destinationipaddress}:' '{destinationport}' ).format(**newmessage['details']) return (newmessage, metadata) if logtype == 'sip': if 'status_msg' not in newmessage['details']: newmessage['details']['status_msg'] = 'unknown' if 'uri' not in newmessage['details']: newmessage['details']['uri'] = 'unknown' if 'method' not in newmessage['details']: newmessage['details']['method'] = 'unknown' newmessage['summary'] = ( 'SIP: {sourceipaddress} -> ' '{destinationipaddress}:' '{destinationport} ' 'method {method} ' 'status {status_msg}' ).format(**newmessage['details']) return (newmessage, metadata) if logtype == 'software': newmessage['details']['parsed_version'] = {} if 'name' not in newmessage['details']: newmessage['details']['name'] = 'unparsed' if 'software_type' not in newmessage['details']: newmessage['details']['software_type'] = 'unknown' if 'host' not in newmessage['details']: newmessage['details'] = '' if 'version.addl' in newmessage['details']: newmessage['details']['parsed_version']['addl'] = newmessage['details']['version.addl'] del(newmessage['details']['version.addl']) if 'version.major' in newmessage['details']: newmessage['details']['parsed_version']['major'] = newmessage['details']['version.major'] del(newmessage['details']['version.major']) if 'version.minor' in newmessage['details']: newmessage['details']['parsed_version']['minor'] = newmessage['details']['version.minor'] del(newmessage['details']['version.minor']) if 'version.minor2' in newmessage['details']: newmessage['details']['parsed_version']['minor2'] = newmessage['details']['version.minor2'] del(newmessage['details']['version.minor2']) if 'version.minor3' in newmessage['details']: newmessage['details']['parsed_version']['minor3'] = newmessage['details']['version.minor3'] del(newmessage['details']['version.minor3']) newmessage['summary'] = ( 'Found {software_type} software ' 'on {host}' ).format(**newmessage['details']) return (newmessage, metadata) if logtype == 'socks': if 'version' not in newmessage['details']: newmessage['details']['version'] = '0' if 'status' not in newmessage['details']: newmessage['details']['status'] = 'unknown' newmessage['summary'] = ( 'SOCKSv{version}: ' '{sourceipaddress} -> ' '{destinationipaddress}:' '{destinationport} ' 'status {status}' ).format(**newmessage['details']) return (newmessage, metadata) if logtype == 'dce_rpc': if 'endpoint' not in newmessage['details']: newmessage['details']['endpoint'] = 'unknown' if 'operation' not in newmessage['details']: newmessage['details']['operation'] = 'unknown' newmessage['summary'] = ( 'DCERPC: {sourceipaddress} -> ' '{destinationipaddress}:' '{destinationport}' ).format(**newmessage['details']) return (newmessage, metadata) if logtype == 'kerberos': if 'request_type' not in newmessage['details']: newmessage['details']['request_type'] = 'unknown' if 'client' not in newmessage['details']: newmessage['details']['client'] = 'unknown' if 'service' not in newmessage['details']: newmessage['details']['service'] = 'unknown' if 'error_msg' not in newmessage['details']: newmessage['details']['error_msg'] = '' newmessage['summary'] = ( '{sourceipaddress} -> ' '{destinationipaddress}:' '{destinationport} ' 'request {request_type}' ).format(**newmessage['details']) if 'success' in newmessage['details']: newmessage['summary'] += ' success {0}'.format(newmessage['details']['success']) else: newmessage['summary'] += ' success unknown' return (newmessage, metadata) if logtype == 'ntlm': newmessage['details']['ntlm'] = {} if 'domainname' in newmessage['details']: newmessage['details']['ntlm']['domainname'] = newmessage['details']['domainname'] del(newmessage['details']['domainname']) else: newmessage['details']['ntlm']['domainname'] = 'unknown' if 'hostname' in newmessage['details']: newmessage['details']['ntlm']['hostname'] = newmessage['details']['hostname'] del(newmessage['details']['hostname']) else: newmessage['details']['ntlm']['hostname'] = 'unknown' if 'username' in newmessage['details']: newmessage['details']['ntlm']['username'] = newmessage['details']['username'] del(newmessage['details']['username']) else: newmessage['details']['ntlm']['username'] = '******' if 'status' not in newmessage['details']: newmessage['details']['status'] = 'unknown' newmessage['summary'] = ( 'NTLM: {sourceipaddress} -> ' '{destinationipaddress}:' '{destinationport} ' ).format(**newmessage['details']) if 'success' in newmessage['details']: newmessage['summary'] += 'success {0} status {1}'.format(newmessage['details']['success'],newmessage['details']['status']) else: newmessage['summary'] += 'success unknown status {0}'.format(newmessage['details']['status']) return (newmessage, metadata) if logtype == 'smb_files': newmessage['details']['smbtimes'] = {} if 'path' not in newmessage['details']: newmessage['details']['path'] = '' if 'name' not in newmessage['details']: newmessage['details']['name'] = '' if 'action' not in newmessage['details']: newmessage['details']['action'] = '' if 'times.accessed' in newmessage['details']: newmessage['details']['smbtimes']['accessed'] = toUTC(float(newmessage['details']['times.accessed'])).isoformat() del(newmessage['details']['times.accessed']) if 'times.changed' in newmessage['details']: newmessage['details']['smbtimes']['changed'] = toUTC(float(newmessage['details']['times.changed'])).isoformat() del(newmessage['details']['times.changed']) if 'times.created' in newmessage['details']: newmessage['details']['smbtimes']['created'] = toUTC(float(newmessage['details']['times.created'])).isoformat() del(newmessage['details']['times.created']) if 'times.modified' in newmessage['details']: newmessage['details']['smbtimes']['modified'] = toUTC(float(newmessage['details']['times.modified'])).isoformat() del(newmessage['details']['times.modified']) newmessage['summary'] = ( 'SMB file: ' '{sourceipaddress} -> ' '{destinationipaddress}:' '{destinationport} ' '{action}' ).format(**newmessage['details']) return(newmessage, metadata) if logtype == 'smb_mapping': if 'share_type' not in newmessage['details']: newmessage['details']['share_type'] = '' if 'path' not in newmessage['details']: newmessage['details']['path'] = '' newmessage['summary'] = ( 'SMB mapping: ' '{sourceipaddress} -> ' '{destinationipaddress}:' '{destinationport} ' '{share_type}' ).format(**newmessage['details']) return(newmessage, metadata) if logtype == 'snmp': if 'version' not in newmessage['details']: newmessage['details']['version'] = 'Unknown' if 'get_bulk_requests' not in newmessage['details']: newmessage['details']['get_bulk_requests'] = 0 if 'get_requests' not in newmessage['details']: newmessage['details']['get_requests'] = 0 if 'set_requests' not in newmessage['details']: newmessage['details']['set_requests'] = 0 if 'get_responses' not in newmessage['details']: newmessage['details']['get_responses'] = 0 newmessage['details']['getreqestssum'] = '{0}'.format(newmessage['details']['get_bulk_requests'] + newmessage['details']['get_requests']) newmessage['summary'] = ( 'SNMPv{version}: ' '{sourceipaddress} -> ' '{destinationipaddress}:' '{destinationport} ' '({getreqestssum} get / ' '{set_requests} set requests ' '{get_responses} get responses)' ).format(**newmessage['details']) return (newmessage, metadata) if logtype == 'x509': newmessage['details']['certificate'] = {} if 'basic_constraints.ca' in newmessage['details']: newmessage['details']['certificate']['basic_constraints_ca'] = newmessage['details']['basic_constraints.ca'] del(newmessage['details']['basic_constraints.ca']) if 'basic_constraints.path_len' in newmessage['details']: newmessage['details']['certificate']['basic_constraints_path_len'] = newmessage['details']['basic_constraints.path_len'] del(newmessage['details']['basic_constraints.path_len']) if 'certificate.exponent' in newmessage['details']: newmessage['details']['certificate']['exponent'] = newmessage['details']['certificate.exponent'] del(newmessage['details']['certificate.exponent']) if 'certificate.issuer' in newmessage['details']: newmessage['details']['certificate']['issuer'] = newmessage['details']['certificate.issuer'] del(newmessage['details']['certificate.issuer']) if 'certificate.key_alg' in newmessage['details']: newmessage['details']['certificate']['key_alg'] = newmessage['details']['certificate.key_alg'] del(newmessage['details']['certificate.key_alg']) if 'certificate.key_length' in newmessage['details']: newmessage['details']['certificate']['key_length'] = newmessage['details']['certificate.key_length'] del(newmessage['details']['certificate.key_length']) if 'certificate.key_type' in newmessage['details']: newmessage['details']['certificate']['key_type'] = newmessage['details']['certificate.key_type'] del(newmessage['details']['certificate.key_type']) if 'certificate.not_valid_after' in newmessage['details']: newmessage['details']['certificate']['not_valid_after'] = toUTC(float(newmessage['details']['certificate.not_valid_after'])).isoformat() del(newmessage['details']['certificate.not_valid_after']) if 'certificate.not_valid_before' in newmessage['details']: newmessage['details']['certificate']['not_valid_before'] = toUTC(float(newmessage['details']['certificate.not_valid_before'])).isoformat() del(newmessage['details']['certificate.not_valid_before']) if 'certificate.sig_alg' in newmessage['details']: newmessage['details']['certificate']['sig_alg'] = newmessage['details']['certificate.sig_alg'] del(newmessage['details']['certificate.sig_alg']) if 'certificate.subject' in newmessage['details']: newmessage['details']['certificate']['subject'] = newmessage['details']['certificate.subject'] del(newmessage['details']['certificate.subject']) if 'certificate.version' in newmessage['details']: newmessage['details']['certificate']['version'] = newmessage['details']['certificate.version'] del(newmessage['details']['certificate.version']) if 'certificate.serial' in newmessage['details']: newmessage['details']['certificate']['serial'] = newmessage['details']['certificate.serial'] del(newmessage['details']['certificate.serial']) else: newmessage['details']['certificate']['serial'] = '0' newmessage['summary'] = ( 'X509 certificate seen' ).format(**newmessage['details']['certificate']) return (newmessage, metadata) return (newmessage, metadata)