def updateAlert(self, alertRef, alert): """ Update an alert in TheHive to add new fields :param alertRef: The reference of the alert to update :type alertRef: string :param alert: The alert object with new data only, appending to the old data is done in this function :type alert: thehive4py.models.Alert :return results: response of the update as a dict :rtype results: dict """ self.logger.info("{}.updateAlert starts".format(__name__)) res = self.findAlert(Eq("sourceRef", alertRef))[0] oldAlert = Alert(json=res) oldAlert.id = res['id'] newAlert = Alert(json=json.loads(alert.jsonify())) # update description oldAlert.description += newAlert.description # update severity oldAlert.severity = max(oldAlert.severity, newAlert.severity) # check for new artifacts, append new ones to old alert oldArtifacts_type = [] oldArtifacts_data = [] for artifact in oldAlert.artifacts: oldArtifacts_type.append(artifact.dataType) oldArtifacts_data.append(artifact.data) for artifact in newAlert.artifacts: if artifact.dataType not in oldArtifacts_type or artifact.data not in oldArtifacts_data: oldAlert.append(artifact) # append new tags for tag in newAlert.tags: if tag not in oldAlert.tags: oldAlert.tags.append(tag) # update alert ret = self.theHiveApi.update_alert( oldAlert.id, oldAlert, fields=['description', 'tags', 'severity', 'artifacts']) if ret.status_code == 200 or ret.status_code == 201: return ret.json() else: self.logger.error('Alert update failed') raise ValueError(json.dumps(ret.json(), indent=4, sort_keys=True))
def build_alert(incident, observables, ignored_tags=None): """ Convert FireEye alert into a TheHive Alert :param incident: Incident from FE :type incident: dict :param observables: observables from FE :type observables: dict :return: Thehive alert :rtype: thehive4py.models Alerts """ a = Alert(title="{}".format(incident.get('title')), tlp=2, severity=th_severity(incident.get('riskRating', 'NONE')), description=description_to_markdown(incident), type=incident.get('intelligenceType', None), tags=th_alert_tags(incident, ignored_tags), caseTemplate=TheHive['template'], source="FireEye", sourceRef=incident.get('reportId'), artifacts=build_observables(observables)) logging.debug("build_alert: alert built for FE id #{}".format( incident.get('reportId'))) return a
def build_alert(incident, observables, thumbnail): """ Convert DigitalShadows alert into a TheHive Alert :param incident: Incident from DS :type incident: dict :param observables: observables from DS :type observables: dict :type thumbnail: str :return: Thehive alert :rtype: thehive4py.models Alerts """ a = Alert(title="{}".format(incident.get('title')), tlp=2, severity=th_severity(incident.get('severity')), description=ds2markdown(incident, thumbnail).thdescription, type=incident.get('type'), tags=th_alert_tags(incident), caseTemplate=TheHive['template'], source="DigitalShadows", sourceRef=str(incident.get('id')), artifacts=build_observables(observables) ) logging.debug("build_alert: alert built for DS id #{}".format(incident.get('id'))) return a
def generate_alert(format_alt, artifacts_dict, w_alert): #generate alert sourceRef sourceRef = str(uuid.uuid4())[0:6] artifacts = [] if 'agent' in w_alert.keys(): if 'ip' not in w_alert['agent'].keys(): w_alert['agent']['ip'] = 'no agent ip' else: w_alert['agent'] = {'id': 'no agent id', 'name': 'no agent name'} for key, value in artifacts_dict.items(): for val in value: artifacts.append(AlertArtifact(dataType=key, data=val)) alert = Alert( title=w_alert['rule']['description'], tlp=2, tags=[ 'wazuh', 'rule=' + w_alert['rule']['id'], 'agent_name=' + w_alert['agent']['name'], 'agent_id=' + w_alert['agent']['id'], 'agent_ip=' + w_alert['agent']['ip'], ], description=format_alt, type='wazuh_alert', source='wazuh', sourceRef=sourceRef, artifacts=artifacts, ) return alert
def create_the_hive_alert(source, path, tag): tags = list(r_serv_metadata.smembers('tag:'+path)) artifacts = [ AlertArtifact( dataType='uuid-ail', data=r_serv_db.get('ail:uuid') ), AlertArtifact( dataType='file', data=path, tags=tags ) ] l_tags = tag.split(',') # Prepare the sample Alert sourceRef = str(uuid.uuid4())[0:6] alert = Alert(title='AIL Leak', tlp=3, tags=l_tags, description='infoleak', type='ail', source=source, sourceRef=sourceRef, artifacts=artifacts) # Create the Alert id = None try: response = HiveApi.create_alert(alert) if response.status_code == 201: #print(json.dumps(response.json(), indent=4, sort_keys=True)) print('Alert Created') print('') id = response.json()['id'] else: print('ko: {}/{}'.format(response.status_code, response.text)) return 0 except: print('hive connection error')
def prepare_alert(self, content, alert_type): """ convert a QRadar alert into a TheHive alert :param content: QRadar Alert :type content: dict :return: Thehive alert :rtype: thehive4py.models Alerts """ #Defined tags that should be present in the alert case_tags = ["src:QRadar"] ''' CHANGE NOT DONE ''' ''' Added APR and RESPC alert type. Still needs a case template and argument adjustment (severity, source etc). Importing the Alert function arguments from config file Please do not forget the change on line 290''' if alert_type in ["lse", "apr", "respc"]: case_tags.append(alert_type) #Build alert as object defined in TheHive4Py models alert = Alert(title=content.get("title"), tlp=2, tags=case_tags, severity=int(content.get('severity', self.ReportConfig[alert_type]['severity'])), description=content.get('description'), type='{} report'.format(alert_type.upper()), source='QRadar', caseTemplate=self.ReportConfig[alert_type]['case_template'], sourceRef="qrpt-{}-{}".format(alert_type, str(self.current_time))) self.logger.info("Alert built for {}".format(self.ReportConfig[alert_type]['description'])) self.logger.debug(str(alert)) return alert
def prepare_alert(content, thumbnails): """ convert a ZeroFOX alert into a TheHive alert :param incident: Zerofox Alert :type incident: dict :type thumbnails: dict :return: Thehive alert :rtype: thehive4py.models Alerts """ case_tags = ["src:ZEROFOX"] case_tags = add_tags(case_tags, [ "Type={}".format(content.get("alert_type")), "Network={}".format(content.get("network")), "Entity={}".format(content.get("entity", {}).get("name", "-")), "Id={}".format(content.get('id')) ]) alert = Alert(title=th_title(content), tlp=2, tags=case_tags, severity=th_severity(content.get('severity', "3")), description=th_case_description(content, thumbnails), type='{}'.format(content.get('alert_type')), source='Zerofox', caseTemplate=TheHive['template'], sourceRef=str(content.get('id')), artifacts=prepare_artifacts(content)) logging.debug("prepare_alert: alert built for \ ZF id #{}".format(content.get('id'))) return alert
def sendHiveAlert(title, tlp, tags, description, sourceRef, artifact_string): tlp = int(parser.get('hive', 'hive_tlp')) hive_api = hiveInit() newtags = tags.strip('][').replace("'", "").split(', ') description = description.strip('"') artifacts = json.loads(artifact_string) # Build alert hivealert = Alert(title=title, tlp=tlp, tags=newtags, description=description, type='external', source='SecurityOnion', sourceRef=sourceRef, artifacts=artifacts) # Send it off response = hive_api.create_alert(hivealert) if response.status_code == 201: print(json.dumps(response.json(), indent=4, sort_keys=True)) print('') else: print('ko: {}/{}'.format(response.status_code, response.text)) sys.exit(0) # Redirect to TheHive instance return redirect(hive_url + '/index.html#!/alert/list')
def send_thehive_message(rule, message, asset, description): print("send_thehive_message:", rule, message, asset, description) thehive_url = Setting.objects.get(key="alerts.endpoint.thehive.url") thehive_apikey = Setting.objects.get(key="alerts.endpoint.thehive.apikey") alert_message = "[Alert][Rule={}]{}".format(rule.title, message) api = TheHiveApi(thehive_url.value, thehive_apikey.value) sourceRef = str(uuid.uuid4())[0:6] rule_severity = 0 if rule.severity == "Low": rule_severity = 1 elif rule.severity == "Medium": rule_severity = 2 elif rule.severity == "High": rule_severity = 3 tlp = 0 if asset.criticity == "low": tlp = 1 elif asset.criticity == "medium": tlp = 2 elif asset.criticity == "high": tlp = 3 if asset: artifacts = [AlertArtifact(dataType=asset.type, data=asset.value)] try: alert = Alert( title=alert_message, tlp=tlp, severity=rule_severity, tags=['src:PatrOwl'], description=description, type='external', source='patrowl', sourceRef=sourceRef, artifacts=artifacts) response = api.create_alert(alert) if response.status_code == 201: alert_id = response.json()['id'] # todo: track theHive alerts Event.objects.create( message="[Rule][send_thehive_message()] " "Alert sent to TheHive with message '{}' (alert id: {})".format(message, alert_id), type="DEBUG", severity="DEBUG" ) else: Event.objects.create( message="[Rule][send_thehive_message()] Unable to send " "alert to TheHive with message ='{}'".format(message), type="ERROR", severity="ERROR" ) except Exception: Event.objects.create( message="[Rule][send_thehive_message()] Unable to send alert " "to TheHive with message ='{}'".format(message), type="ERROR", severity="ERROR")
def alert(self, matches): connection_details = self.rule['hive_connection'] api = TheHiveApi( '{hive_host}:{hive_port}'.format(**connection_details), connection_details.get('hive_apikey',''), proxies=connection_details.get('hive_proxies', {'http': '', 'https': ''}), cert=connection_details.get('hive_verify', False)) for match in matches: context = {'rule': self.rule, 'match': match} artifacts = [] for mapping in self.rule.get('hive_observable_data_mapping', []): for observable_type, match_data_key in mapping.iteritems(): try: match_data_keys = re.findall(r'\{match\[([^\]]*)\]', match_data_key) if all([True for k in match_data_keys if k in context['match']]): artifacts.append(AlertArtifact(dataType=observable_type, data=match_data_key.format(**context))) except KeyError: raise KeyError('\nformat string\n{}\nmatch data\n{}'.format(match_data_key, context)) alert_config = { 'artifacts': artifacts, 'sourceRef': str(uuid.uuid4())[0:6], 'title': '{rule[name]}'.format(**context) } alert_config.update(self.rule.get('hive_alert_config', {})) for alert_config_field, alert_config_value in alert_config.iteritems(): if alert_config_field == 'customFields': custom_fields = CustomFieldHelper() for cf_key, cf_value in alert_config_value.iteritems(): try: func = getattr(custom_fields, 'add_{}'.format(cf_value['type'])) except AttributeError: raise Exception('unsupported custom field type {}'.format(cf_value['type'])) value = cf_value['value'].format(**context) func(cf_key, value) alert_config[alert_config_field] = custom_fields.build() elif isinstance(alert_config_value, basestring): alert_config[alert_config_field] = alert_config_value.format(**context) elif isinstance(alert_config_value, (list, tuple)): formatted_list = [] for element in alert_config_value: try: formatted_list.append(element.format(**context)) except: formatted_list.append(element) alert_config[alert_config_field] = formatted_list alert = Alert(**alert_config) response = api.create_alert(alert) if response.status_code != 201: raise Exception('alert not successfully created in TheHive\n{}'.format(response.text))
def create_alarm(self, title, source_ref=None, description='N/A', alert_type='external', source='LogRhythm', iocs=None, additional_fields=None, additional_tags=None, tlp=TLP.AMBER, pap=PAP.AMBER, severity=HiveSeverity.MEDIUM): if source_ref is None: source_ref = str(uuid.uuid4())[0:6] alert_tags = self.alert_tags.copy() if additional_tags is not None: for additional_tag in additional_tags: alert_tags.append(additional_tag) custom_fields_helper = CustomFieldHelper() if additional_fields is not None: for field in additional_fields: custom_fields_helper.add_string(field['name'], field['value']) custom_fields = custom_fields_helper.build() artifacts = list() if iocs is not None: for ioc in iocs: artifacts.append( AlertArtifact(dataType=ioc['type'].value, data=ioc['value'])) hive_alert = Alert(title=title, tlp=tlp.value, tags=alert_tags, description=description, type=alert_type, source=source, sourceRef=source_ref, pap=pap.value, artifacts=artifacts, customFields=custom_fields, severity=severity.value) response = self.api.create_alert(hive_alert) if response.status_code == 201: print('Alerta Creada Exitosamente') print(json.dumps(response.json(), indent=4, sort_keys=True)) else: print('Error') print(response.text) return response.json()
def createAlert(self, alert): self.logger.info('%s.createAlert starts', __name__) response = self.theHiveApi.create_alert(alert) if response.status_code == 201: esAlertId = response.json()['id'] createdAlert = Alert(json=self.theHiveApi.get_alert(esAlertId).json()) return createdAlert else: self.logger.error('Alert creation failed') raise ValueError(json.dumps(response.json(), indent=4, sort_keys=True))
def CreateHiveAlertFromSentinel(api, title, description, incidentnumber, severity, source, artifacts, alertIds, incidentURL): tags = [] if severity == "Low": theHiveSeverity = 1 elif severity == "Informational": theHiveSeverity = 1 elif severity == "Medium": theHiveSeverity = 2 elif severity == "High": theHiveSeverity = 3 alertIdsStr = ' '.join(map(str, alertIds)) customFields = CustomFieldHelper() customFields.add_number('sentinelIncidentNumber', incidentnumber) customFields.add_string('alertIds', alertIdsStr) customFields.add_string('incidentURL', incidentURL) customFields = customFields.build() alert = Alert(title=title, tlp=2, tags=tags, description=description, type='Sentinel', severity=theHiveSeverity, source='Sentinel:' + source, customFields=customFields, sourceRef="Sentinel" + str(incidentnumber), artifacts=artifacts) # Create the Alert response = api.create_alert(alert) if response.status_code == 201: logging.info( 'Alert created: ' + 'Sentinel' + str(incidentnumber) + ': ' + source + ': ' + title + '. StatusCode: {}/{}'.format(response.status_code, response.text)) elif (response.status_code == 400 and response.json()['type'] == "ConflictError"): logging.info( 'Duplicate alert: ' + 'Sentinel' + str(incidentnumber) + ': ' + source + ': ' + title + '. StatusCode: {}/{}'.format(response.status_code, response.text)) else: logging.error( 'failed to create alert: ' + source + ' ' + title + ' Sentinel' + str(incidentnumber) + '. StatusCode: {}/{}'.format(response.status_code, response.text)) sys.exit(0)
def send_to_thehive(self, alert_config): connection_details = self.rule['hive_connection'] api = TheHiveApi( '{hive_host}:{hive_port}'.format(**connection_details), connection_details.get('hive_apikey', ''), proxies=connection_details.get('hive_proxies', {'http': '', 'https': ''}), cert=connection_details.get('hive_verify', False)) alert = Alert(**alert_config) response = api.create_alert(alert) if response.status_code != 201: raise Exception('alert not successfully created in TheHive\n{}'.format(response.text))
def sendHiveAlert(title, tlp, tags, description, sourceRef, artifact_string): hive_url = parser.get('hive', 'hive_url') hive_key = parser.get('hive', 'hive_key') hive_verifycert = parser.get('hive', 'hive_verifycert') tlp = int(parser.get('hive', 'hive_tlp')) # Check if verifying cert if 'False' in hive_verifycert: hiveapi = TheHiveApi(hive_url, hive_key, cert=False) else: hiveapi = TheHiveApi(hive_url, hive_key, cert=True) newtags = tags.strip('][').replace("'","").split(', ') artifacts = json.loads(artifact_string) #print(newtags) # Build alert hivealert = Alert( title= title, tlp=tlp, tags=newtags, description=description, type='external', source='SecurityOnion', sourceRef=sourceRef, artifacts=artifacts ) # Send it off response = hiveapi.create_alert(hivealert) if response.status_code == 201: print(json.dumps(response.json(), indent=4, sort_keys=True)) print('') id = response.json()['id'] # If running standalone / eval tell ES that we sent the alert #es_type = 'doc' #es_index = index #es_headers = {'Content-Type' : 'application/json'} #es_data = '{"script" : {"source": "ctx._source.tags.add(params.tag)","lang": "painless","params" : {"tag" : "Sent to TheHive"}}}' #update_es_event = requests.post(es_url + '/' + es_index + '/' + es_type + '/' + esid + '/_update', headers=es_headers, data=es_data) #print(update_es_event.content) else: print('ko: {}/{}'.format(response.status_code, response.text)) sys.exit(0) # Redirect to TheHive instance return redirect(hive_url + '/index.html#!/alert/list')
def craftAlert(self, title, description, severity, date, tags, sourceRef, artifacts_list): self.logger.info('%s.craftAlert starts', __name__) alert = Alert(title=title, description=description, severity=severity, date=date, tags=tags, type='SIEM', source='QRadar', sourceRef=sourceRef, artifacts=artifacts_list) return alert
def makealert(input_title): # Prepare the sample Alert sourceRef = str(uuid.uuid4())[0:6] alert = Alert(title=input_title, tlp=3, tags=['TheHive4Py'], description='N/A', type='external', source='instance1', sourceRef=sourceRef) # Create the Alert response = api.create_alert(alert) print(response.text) print(response.status_code)
def prepare_thehive_alert(cls, event: SocEvent) -> Alert: return Alert( title=cls._get_title(event), type=cls._get_alert_type(event), source=cls._get_alert_source(event), sourceRef=cls._get_alert_source_ref(event), description=cls._get_description(event), customFields=cls.prepare_custom_fields(event), # below only not required attributes date=cls._get_datetime(event), severity=cls._get_severity(event), caseTemplate=cls._get_case_template_name_for_alert(event), tags=cls._prepare_tags(event), # items that will be used in case investigation artifacts=cls._prepare_artifacts(event) )
def prepare_alert(subject, indicatorlevel, emailbody, alerttype, tag): hive_address = ''.join(settings.stored_hive_address[0]) hive_api = ''.join(settings.stored_api_key[0]) #Define the connection to thehive installation (including the generated API key). api = TheHiveApi(hive_address, hive_api, None, {'http': '', 'https': ''}) # Prepare the sample Alert print("Preparing the alert for", alerttype) sourceRef = str(uuid.uuid4())[0:6] alert = Alert(title=subject, tlp=indicatorlevel, tags=[tag], description=emailbody, type=alerttype, source='instance1', sourceRef=sourceRef, artifacts="") # Create the Alert print('Create Alert') print('-----------------------------') id = None response = api.create_alert(alert) if response.status_code == 201: print("Alert created sucessfully!") # print(json.dumps(response.json(), indent=4, sort_keys=True)) # print('') id = response.json()['id'] else: print("Unable to create alert") # print('ko: {}/{}'.format(response.status_code, response.text)) sys.exit(0) # Get all the details of the created alert print('Get created alert {}'.format(id)) print('-----------------------------') response = api.get_alert(id) if response.status_code == requests.codes.ok: # print(json.dumps(response.json(), indent=4, sort_keys=True)) print('') else: print("Error retreiving the alert!") print('ko: {}/{}'.format(response.status_code, response.text))
def craftAlert(self, title, description, severity, date, tags, tlp, status, type, source, sourceRef, artifacts, caseTemplate): self.logger.info('%s.craftAlert starts', __name__) alert = Alert(title=title, description=description, severity=severity, date=date, tags=tags, tlp=tlp, type=type, source=source, sourceRef=sourceRef, artifacts=artifacts, caseTemplate=caseTemplate) return alert
def alert(self, matches): connection_details = self.rule['hive_connection'] api = TheHiveApi( '{hive_host}:{hive_port}'.format(**connection_details), '{hive_username}'.format(**connection_details), '{hive_password}'.format(**connection_details), connection_details.get('hive_proxies', {'http': '', 'https': ''})) for match in matches: context = {'rule': self.rule, 'match': match} artifacts = [] for mapping in self.rule.get('hive_observable_data_mapping', []): for observable_type, match_data_key in mapping.iteritems(): try: artifacts.append(AlertArtifact(dataType=observable_type, data=match_data_key.format(**context))) except KeyError: raise KeyError('\nformat string\n{}\nmatch data\n{}'.format(match_data_key, context)) alert_config = { 'artifacts': artifacts, 'sourceRef': str(uuid.uuid4())[0:6], 'title': '{rule[index]}_{rule[name]}'.format(**context) } alert_config.update(self.rule.get('hive_alert_config', {})) for alert_config_field, alert_config_value in alert_config.iteritems(): if isinstance(alert_config_value, basestring): alert_config[alert_config_field] = alert_config_value.format(**context) elif isinstance(alert_config_value, (list, tuple)): formatted_list = [] for element in alert_config_value: try: formatted_list.append(element.format(**context)) except: formatted_list.append(element) alert_config[alert_config_field] = formatted_list alert = Alert(**alert_config) response = api.create_alert(alert) if response.status_code != 201: raise Exception('alert not successfully created in TheHive\n{}'.format(response.text))
def craftAlert(self, title, description, severity, date, tags, tlp, status, type, source, sourceRef, artifacts, caseTemplate): from thehive4py.models import Case, CaseTask, CaseTaskLog, CaseObservable, Alert alert = Alert(title=title, description=description, severity=severity, date=date, tags=tags, tlp=tlp, type=type, source=source, sourceRef=sourceRef, artifacts=artifacts, caseTemplate=caseTemplate) return alert
def create_the_hive_alert(source, item_id, tag): # # TODO: check items status (processed by all modules) # # TODO: add item metadata: decoded content, link to auto crawled content, pgp correlation, cryptocurrency correlation... # # # TODO: description, add AIL link:show items ? tags = list(r_serv_metadata.smembers('tag:{}'.format(item_id))) artifacts = [ AlertArtifact(dataType='uuid-ail', data=r_serv_db.get('ail:uuid')), AlertArtifact(dataType='file', data=item_basic.get_item_filepath(item_id), tags=tags) ] # Prepare the sample Alert sourceRef = str(uuid.uuid4())[0:6] alert = Alert(title='AIL Leak', tlp=3, tags=tags, description='AIL Leak, triggered by {}'.format(tag), type='ail', source=source, sourceRef=sourceRef, artifacts=artifacts) # Create the Alert id = None try: response = HiveApi.create_alert(alert) if response.status_code == 201: #print(json.dumps(response.json(), indent=4, sort_keys=True)) print('Alert Created') print('') id = response.json()['id'] else: print('ko: {}/{}'.format(response.status_code, response.text)) return 0 except: print('hive connection error')
def createHiveAlert(esid): search = getHits(esid) #Hive Stuff #es_url = parser.get('es', 'es_url') hive_url = parser.get('hive', 'hive_url') hive_key = parser.get('hive', 'hive_key') hive_verifycert = parser.get('hive', 'hive_verifycert') tlp = int(parser.get('hive', 'hive_tlp')) # Check if verifying cert if 'False' in hive_verifycert: api = TheHiveApi(hive_url, hive_key, cert=False) else: api = TheHiveApi(hive_url, hive_key, cert=True) #if hits > 0: for result in search['hits']['hits']: # Get initial details message = result['_source']['message'] description = str(message) sourceRef = str(uuid.uuid4())[0:6] tags=["SecurityOnion"] artifacts=[] id = None host = str(result['_index']).split(":")[0] index = str(result['_index']).split(":")[1] event_type = result['_source']['event_type'] if 'source_ip' in result['_source']: src = str(result['_source']['source_ip']) if 'destination_ip' in result['_source']: dst = str(result['_source']['destination_ip']) #if 'source_port' in result['_source']: # srcport = result['_source']['source_port'] #if 'destination_port' in result['_source']: # dstport = result['_source']['destination_port'] # NIDS Alerts if 'snort' in event_type: alert = result['_source']['alert'] category = result['_source']['category'] sensor = result['_source']['interface'] tags.append("nids") tags.append(category) title=alert # Add artifacts artifacts.append(AlertArtifact(dataType='ip', data=src)) artifacts.append(AlertArtifact(dataType='ip', data=dst)) artifacts.append(AlertArtifact(dataType='other', data=sensor)) # Bro logs elif 'bro' in event_type: _map_key_type ={ "conn": "Connection", "dhcp": "DHCP", "dnp3": "DNP3", "dns": "DNS", "files": "Files", "ftp": "FTP", "http": "HTTP", "intel": "Intel", "irc": "IRC", "kerberos": "Kerberos", "modbus": "Modbus", "mysql": "MySQL", "ntlm": "NTLM", "pe": "PE", "radius": "RADIUS", "rdp": "RDP", "rfb": "RFB", "sip" : "SIP", "smb": "SMB", "smtp": "SMTP", "snmp": "SNMP", "ssh": "SSH", "ssl": "SSL", "syslog": "Syslog", "weird": "Weird", "x509": "X509" } def map_key_type(indicator_type): ''' Maps a key type to use in the request URL. ''' return _map_key_type.get(indicator_type) bro_tag = event_type.strip('bro_') bro_tag_title = map_key_type(bro_tag) title= str('New Bro ' + bro_tag_title + ' record!') if 'source_ip' in result['_source']: artifacts.append(AlertArtifact(dataType='ip', data=src)) if 'destination_ip' in result['_source']: artifacts.append(AlertArtifact(dataType='ip', data=dst)) if 'sensor_name' in result['_source']: sensor = str(result['_source']['sensor_name']) artifacts.append(AlertArtifact(dataType='other', data=sensor)) if 'uid' in result['_source']: uid = str(result['_source']['uid']) title= str('New Bro ' + bro_tag_title + ' record! - ' + uid) artifacts.append(AlertArtifact(dataType='other', data=uid)) if 'fuid' in result['_source']: fuid = str(result['_source']['fuid']) title= str('New Bro ' + bro_tag_title + ' record! - ' + fuid) artifacts.append(AlertArtifact(dataType='other', data=fuid)) if 'id' in result['_source']: fuid = str(result['_source']['id']) title= str('New Bro ' + bro_tag_title + ' record! - ' + fuid) artifacts.append(AlertArtifact(dataType='other', data=fuid)) tags.append('bro') tags.append(bro_tag) # Wazuh/OSSEC logs elif 'ossec' in event_type: agent_name = result['_source']['agent']['name'] if 'description' in result['_source']: ossec_desc = result['_source']['description'] else: ossec_desc = result['_source']['full_log'] if 'ip' in result['_source']['agent']: agent_ip = result['_source']['agent']['ip'] artifacts.append(AlertArtifact(dataType='ip', data=agent_ip)) artifacts.append(AlertArtifact(dataType='other', data=agent_name)) else: artifacts.append(AlertArtifact(dataType='other', data=agent_name)) title= ossec_desc tags.append("wazuh") elif 'sysmon' in event_type: if 'ossec' in result['_source']['tags']: agent_name = result['_source']['agent']['name'] agent_ip = result['_source']['agent']['ip'] ossec_desc = result['_source']['full_log'] artifacts.append(AlertArtifact(dataType='ip', data=agent_ip)) artifacts.append(AlertArtifact(dataType='other', data=agent_name)) title= "New Sysmon Event! - " + agent_name tags.append("wazuh") else: title = "New " + event_type + " Event From Security Onion" # Build alert hivealert = Alert( title= title, tlp=tlp, tags=tags, description=description, type='external', source='SecurityOnion', sourceRef=sourceRef, artifacts=artifacts ) # Send it off response = api.create_alert(hivealert) if response.status_code == 201: print(json.dumps(response.json(), indent=4, sort_keys=True)) print('') id = response.json()['id'] # If running standalone / eval tell ES that we sent the alert #es_type = 'doc' #es_index = index #es_headers = {'Content-Type' : 'application/json'} #es_data = '{"script" : {"source": "ctx._source.tags.add(params.tag)","lang": "painless","params" : {"tag" : "Sent to TheHive"}}}' #update_es_event = requests.post(es_url + '/' + es_index + '/' + es_type + '/' + esid + '/_update', headers=es_headers, data=es_data) #print(update_es_event.content) else: print('ko: {}/{}'.format(response.status_code, response.text)) sys.exit(0)
def create_alert(helper, thehive_api, alert_args): """ This function is used to create the alert using the API, settings and search results """ # iterate through each row, cleaning multivalue fields # and then adding the attributes under same alert key # this builds the dict alerts app_name = "TA-thehive-cortex" data_type = get_datatype_dict(helper, app_name) custom_field_type = get_customField_dict(helper, app_name) alert_reference = 'SPK' + str(int(time.time())) helper.log_debug( "[CAA-THCA-60] alert_reference: {}".format(alert_reference)) alerts = dict() events = helper.get_events() for row in events: # Initialize values artifacts = [] artifactTags = [] artifactMessage = '' customFields = dict() sourceRef = alert_reference alert = dict() # Splunk makes a bunch of dumb empty multivalue fields # replace value by multivalue if required helper.log_debug("[CAA-THCA-65] Row before pre-processing: " + str(row)) for key, value in row.items(): if not key.startswith("__mv_") and "__mv_" + key in row and row[ "__mv_" + key] not in [None, '']: row[key] = [ e[1:len(e) - 1] for e in row["__mv_" + key].split(";") ] # we filter those out here row = { key: value for key, value in row.items() if not key.startswith("__mv_") and key not in ["rid"] } helper.log_debug("[CAA-THCA-66] Row after pre-processing: " + str(row)) # find the field name used for a unique identifier if alert_args['unique_id_field'] in row: newSource = str(row[alert_args['unique_id_field']]) if newSource not in [None, '']: # grabs that field's value and assigns it to our sourceRef sourceRef = newSource helper.log_debug("[CAA-THCA-70] sourceRef: {} ".format(sourceRef)) if 'th_inline_tags' in row: # grabs that field's value and assigns it to artifactTags = list(str(row.pop("th_inline_tags")).split(",")) # check if the field th_msg exists and strip it from the row. # The value will be used as message attached to artifacts if 'th_msg' in row: # grabs that field's value and assigns it to artifactMessage = str(row.pop("th_msg")) helper.log_debug( "[CAA-THCA-75] artifact message: {} ".format(artifactMessage)) helper.log_debug( "[CAA-THCA-76] artifact tags: {} ".format(artifactTags)) # check if artifacts have been stored for this sourceRef. # If yes, retrieve them to add new ones from this row if sourceRef in alerts: alert = alerts[sourceRef] artifacts = list(alert["artifacts"]) customFields = dict(alert['customFields']) # check if title contains a field name instead of a string. # if yes, strip it from the row and assign value to title alert["title"] = extract_field(helper, row, alert_args["title"]) # check if description contains a field name instead of a string. # if yes, strip it from the row and assign value to description alert["description"] = extract_field(helper, row, alert_args["description"]) # find the field name used for a valid timestamp # and strip it from the row if alert_args['timestamp'] in row: newTimestamp = str(int(float(row.pop(alert_args['timestamp'])))) helper.log_debug( "[CAA-THCA-80] new Timestamp from row: {} ".format( newTimestamp)) epoch10 = re.match("^[0-9]{10}$", newTimestamp) epoch13 = re.match("^[0-9]{13}$", newTimestamp) if epoch13 is not None: alert['timestamp'] = int(float(newTimestamp)) elif epoch10 is not None: alert['timestamp'] = int(float(newTimestamp)) * 1000 helper.log_debug("[CAA-THCA-85] alert timestamp: {} ".format( alert['timestamp'])) else: alert['timestamp'] = alert_args['timestamp'] # now we take those KV pairs to add to dict for key, value in row.items(): cTags = artifactTags[:] if value != "": helper.log_debug( '[CAA-THCA-90] field to process: {}'.format(key)) artifact_key = '' cTLP = '' if ':' in key: helper.log_debug( '[CAA-THCA-91] composite fieldvalue: {}'.format(key)) dType = key.split(':', 1) key = str(dType[0]) # extract TLP at observable level # it is on letter W G A or R appended to field name observable_tlp_check = re.match("^(W|G|A|R)$", str(dType[1])) if observable_tlp_check is not None: cTLP = OBSERVABLE_TLP[dType[1]] cTags.append(OBSERVABLE_TLP[str(cTLP)]) else: cTags.append(str(dType[1]).replace(" ", "_")) if key in data_type: helper.log_debug( '[CAA-THCA-95] key is an artifact: {} '.format(key)) artifact_key = data_type[key] elif key in custom_field_type: helper.log_debug( '[CAA-THCA-96] key is a custom field: {} '.format(key)) # expected types are `string`, `boolean`, `number` (only TH3), `date`, `integer` (only TH4), `float` (only TH4) custom_field_check = False custom_field = dict() custom_field['order'] = len(customFields) custom_type = custom_field_type[key] if custom_type == 'string': custom_field_check = True custom_field[custom_type] = str(value) elif custom_type == 'boolean': is_True = re.match("^(1|y|Y|t|T|true|True)$", value) is_False = re.match("^(0|n|N|f|F|false|False)$", value) if is_True is not None: custom_field_check = True custom_field[custom_type] = True elif is_False is not None: custom_field_check = True custom_field[custom_type] = False elif custom_type == 'number': # for TheHive3 only is_integer = re.match("^[0-9]+$", value) if is_integer is not None: custom_field_check = True custom_field[custom_type] = int(value) else: try: number = float(value) custom_field_check = True custom_field[custom_type] = number except ValueError: pass elif custom_type == 'integer': # for TheHive4 only try: number = int(value) custom_field_check = True custom_field[custom_type] = number except ValueError: pass elif custom_type == 'float': # for TheHive4 only try: number = float(value) custom_field_check = True custom_field[custom_type] = number except ValueError: pass elif custom_type == 'date': epoch10 = re.match("^[0-9]{10}$", value) epoch13 = re.match("^[0-9]{13}$", value) if epoch13 is not None: custom_field_check = True custom_field[custom_type] = int(value) elif epoch10 is not None: custom_field_check = True custom_field[custom_type] = int(value) * 1000 if custom_field_check is True: customFields[key] = custom_field elif alert_args['scope'] is False: helper.log_debug( '[CAA-THCA-105] key is added as another artifact (scope is False): {} ' .format(key)) artifact_key = 'other' if artifact_key not in [None, '']: helper.log_debug( "[CAA-THCA-106] Processing artifact key: " + str(artifact_key) + " (" + str(value) + ") (" + artifactMessage + ")") cTags.append('field:' + str(key)) if isinstance(value, list): # was a multivalue field helper.log_debug( '[CAA-THCA-107] value is not a simple string: {} '. format(value)) for val in value: if val != "": artifact = dict(dataType=artifact_key, data=str(val), message=artifactMessage, tags=cTags) if cTLP != '': artifact['tlp'] = cTLP helper.log_debug( "[CAA-THCA-110] new artifact is {}".format( artifact)) if artifact not in artifacts: artifacts.append(artifact) else: artifact = dict(dataType=artifact_key, data=str(value), message=artifactMessage, tags=cTags) if cTLP != '': artifact['tlp'] = cTLP if artifact not in artifacts: artifacts.append(artifact) if artifacts: alert['artifacts'] = list(artifacts) alert['customFields'] = customFields alerts[sourceRef] = alert helper.log_debug("[CAA-THCA-115] Artifacts found for an alert: " + str(artifacts)) else: helper.log_debug( "[CAA-THCA-116] No artifact found for an alert: " + str(alert)) # actually send the request to create the alert; fail gracefully for srcRef in alerts.keys(): # Create the Alert object alert = Alert(title=alerts[srcRef]['title'], date=int(alerts[srcRef]['timestamp']), description=alerts[srcRef]['description'], tags=alert_args['tags'], severity=alert_args['severity'], tlp=alert_args['tlp'], pap=alert_args['pap'], type=alert_args['type'], artifacts=alerts[srcRef]['artifacts'], customFields=alerts[srcRef]['customFields'], source=alert_args['source'], caseTemplate=alert_args['caseTemplate'], sourceRef=srcRef) helper.log_debug("[CAA-THCA-120] Processing alert: " + alert.jsonify()) # Get API and create the alert response = thehive_api.create_alert(alert) if response.status_code in (200, 201, 204): # log response status helper.log_info( "[CAA-THCA-125] INFO theHive alert is successfully created. " "url={}, HTTP status={}".format(thehive_api.url, response.status_code)) else: # somehow we got a bad response code from thehive helper.log_error( "[CAA-THCA-126] ERROR theHive alert creation has failed. " "url={}, data={}, HTTP Error={}, content={}".format( thehive_api.url, alert.jsonify(), response.status_code, response.text))
def build_alert(detection, id): """ Convert CrowdStrike Falcon alert into TheHive Alerts :param detection: Detection from CrowdStrike Alert :type detection: dict :param observables: observables from CrowdStrike Alert :type observables: dict :return: TheHive alert :rtype: thehive4py.models Alerts """ meta = detection['data']['meta'] resources = detection['data']['resources'][0] status = detection['status'] behaviors = resources['behaviors'][0] device = resources['device'] detection_title = '{} {} {} {}'.format( 'CROWDSTRIKE ALERT - ', resources.get('max_severity_displayname'), 'Severity', behaviors['scenario'].replace("_", " ")) tlp = 2 artifacts = [ AlertArtifact(dataType='filename', data=behaviors.get('filename'), message="File name of the triggering process"), AlertArtifact(dataType='hash', data=behaviors.get('sha256'), tags=['sha256'], message="SHA256 of the triggering process", ioc=True), AlertArtifact(dataType='other', data=behaviors.get('user_name'), tags=['target-user'], message="The user's name", tlp=3), AlertArtifact(dataType='other', data=behaviors.get('user_id'), message="Users SID in Windows", tlp=3), AlertArtifact(dataType='hash', data=behaviors.get('md5'), tags=['md5'], message="MD5 of the triggering process", ioc=True), AlertArtifact(dataType='other', data=behaviors.get('cmdline'), message="Command Line of the triggering process"), AlertArtifact(dataType='other', data=behaviors.get('ioc_source'), message="Source that triggered an IOC detection"), AlertArtifact(dataType=th_datatype(behaviors.get('ioc_type')), data=behaviors.get('ioc_value'), message="ioc_value", ioc=True), AlertArtifact(dataType='other', data=device.get('hostname'), tags=['target-machine'], message="Device hostname"), AlertArtifact(dataType='ip', data=device.get('local_ip'), tags=['host-ip'], message="Hosts local ip.", tlp=3), AlertArtifact(dataType='ip', data=device.get('external_ip'), tags=['external-ip'], message="Hosts external ip.") ] a = Alert( title='{} {} {}'.format(detection_title, 'on', device.get('hostname')), tlp=2, severity=th_severity(resources.get('max_severity_displayname')), type="FalconHost Alert", tags=[ "CrowdStrike:Scenario={}".format(behaviors['scenario'].replace( "_", " ").title()), "CrowdStrike:Max Confidence={}".format( resources.get('max_confidence')), "CrowdStrike:Max Severity={}".format( resources.get('max_severity')), device.get('machine_domain'), device.get('hostname'), device.get('local_ip'), behaviors.get('user_name'), behaviors.get('tactic'), behaviors.get('technique') ], caseTemplate=TheHive['template'], source="CrowdStrike", sourceRef=id, artifacts=artifacts, # Switch description to named parameters? description="{0} {1} {2} {3} {4} {5} {6}".format( "#### **" + detection_title + 'on ' + device.get('hostname') + "**", "\n\n --- \n", "#### **SUMMARY**\n\n", "{0} {1} {2} {3} {4} {5} {6} {7}".format( "- **SCENARIO: **" + behaviors.get('scenario') + "\n", "- **SEVERITY: **" + resources.get('max_severity_displayname') + "\n", "- **DETECT TIME: **" + behaviors.get('timestamp') + "\n", "- **HOST: **" + device.get('hostname') + "\n", "- **HOST IP: **" + device.get('local_ip') + "\n", "- **USERNAME: **" + behaviors.get('user_name') + "\n", "- **TACTIC: **" + behaviors.get('tactic') + "\n", "- **TECHNIQUE: **" + behaviors.get('technique') + "\n", ), "\n\n --- \n", "#### **DETECTION DETAILS**\n\n", "{0} {1} {2}".format( "```\n", json.dumps(detection, indent=4, sort_keys=True), "\n```", ), )) return a
def createGRRFlow(esid, flow_name): search = getHits(esid) tlp = int(parser.get('hive', 'hive_tlp')) # Check if verifying cert if 'False' in hive_verifycert: hiveapi = TheHiveApi(hive_url, hive_key, cert=False) else: hiveapi = TheHiveApi(hive_url, hive_key, cert=True) grr_url = parser.get('grr', 'grr_url') grr_user = parser.get('grr', 'grr_user') grr_pass = parser.get('grr', 'grr_pass') grrapi = api.InitHttp(api_endpoint=grr_url, auth=(grr_user, grr_pass)) base64string = '%s:%s' % (grr_user, grr_pass) base64string = base64.b64encode(bytes(base64string, "utf-8")) authheader = "Basic %s" % base64string index_response = requests.get(grr_url, auth=HTTPBasicAuth(grr_user, grr_pass)) csrf_token = index_response.cookies.get("csrftoken") headers = { "Authorization": authheader, "x-csrftoken": csrf_token, "x-requested-with": "XMLHttpRequest" } cookies = {"csrftoken": csrf_token} for result in search['hits']['hits']: result = result['_source'] message = result['message'] description = str(message) info = description if 'source_ip' in result: source_ip = result['source_ip'] if 'destination_ip' in result: destination_ip = result['destination_ip'] for ip in source_ip, destination_ip: search_result = grrapi.SearchClients(ip) grr_result = {} client_id = '' for client in search_result: # Get client id client_id = client.client_id client_last_seen_at = client.data.last_seen_at grr_result[client_id] = client_last_seen_at #flow_name = "ListProcesses" if client_id is None: pass # Process flow and get flow id flow_id = listProcessFlow(client_id, grr_url, headers, cookies, grr_user, grr_pass) # Get status status = checkFlowStatus(client_id, grr_url, flow_id, headers, cookies, grr_user, grr_pass) # Keep checking to see if complete while status != "terminated": time.sleep(15) print( "Flow not yet completed..watiing 15 secs before attempting to check status again..." ) status = checkFlowStatus(client_id, grr_url, flow_id, headers, cookies, grr_user, grr_pass) # If terminated, run the download if status == "terminated": downloadFlowResults(client_id, grr_url, flow_id, headers, cookies, grr_user, grr_pass) #print("Done!") # Run flow via API client #flow_obj = grrapi.Client(client_id) #flow_obj.CreateFlow(name=flow_name) title = "Test Alert with GRR Flow" description = str(message) sourceRef = str(uuid.uuid4())[0:6] tags = ["SecurityOnion", "GRR"] artifacts = [] id = None filepath = "/tmp/soctopus/" + client_id + ".zip" artifacts.append( AlertArtifact(dataType='file', data=str(filepath))) # Build alert hivealert = Alert(title=title, tlp=tlp, tags=tags, description=description, type='external', source='SecurityOnion', sourceRef=sourceRef, artifacts=artifacts) # Send it off response = hiveapi.create_alert(hivealert) if client_id: # Redirect to GRR instance return redirect(grr_url + '/#/clients/' + client_id + '/flows') else: return "No matches found for source or destination ip"
def create_alert(): # Get request JSON content = request.get_json() # Configure logging logging.basicConfig( filename=app.config['LOG_FILE'], filemode='a', format='%(asctime)s - graylog2thehive - %(levelname)s - %(message)s', level=logging.INFO) logging.info(json.dumps(content)) # Configure API api = TheHiveApi(app.config['HIVE_URL'], app.config['API_KEY']) # Configure artifacts artifacts = [] # Configure tags tags = ['graylog'] # Build description body and tags list description = 'Alert Condition: \n' + content['check_result'][ 'triggered_condition']['title'] + '\n\nMatching messages:\n\n' tags = ['graylog'] for message in content['check_result']['matching_messages']: description = description + "\n\n---\n\n**Source:** " + message[ 'source'] + "\n\n**Log URL:** " + app.config[ 'GRAYLOG_URL'] + "/messages/" + message[ 'index'] + "/" + message['id'] + "\n\n" message_flattened = flatten_dict(message) for key in message_flattened.keys(): if key != "message" and key != "source": description = description + "\n**" + key + ":** " + json.dumps( message_flattened[key], ensure_ascii=False) + "\n" # Use any IPs, hashes, URLs, filenames, etc here in place of src_ip and dst_ip to include them as artifacts/observables in your alert if key == "cmd_url2": artifacts.append( AlertArtifact(dataType='url', tags=[key], data=message_flattened[key])) if key == "dest_ip": artifacts.append( AlertArtifact(dataType='ip', tags=[key], data=message_flattened[key])) if key == "MD5" or key == "SHA256": artifacts.append( AlertArtifact(dataType='hash', tags=[key], data=message_flattened[key])) description = description + '\n\n**Raw Message:** \n\n```\n' + json.dumps( message) + '\n```\n---\n' # Prepare alert sourceRef = str(uuid.uuid4())[0:6] alert = Alert(title="Graylog Alert: " + content['stream']['title'] + " - " + content['check_result']['result_description'], tlp=2, tags=tags, description=content['check_result']['result_description'], type='external', source='graylog', artifacts=artifacts, sourceRef=sourceRef) # Create the alert print('Create Alert') print('-----------------------------') id = None response = api.create_alert(alert) if response.status_code == 201: logging.info(json.dumps(response.json(), indent=4, sort_keys=True)) print(json.dumps(response.json(), indent=4, sort_keys=True)) print('') id = response.json()['id'] else: print('ko: {}/{}'.format(response.status_code, response.text)) sys.exit(0) return content['check_result']['result_description']
def submitTheHive(message): '''Create a new case in TheHive based on the email''' # Decode email msg = email.message_from_bytes(message) decode = email.header.decode_header(msg['From'])[0] fromField = str(decode[0]) decode = email.header.decode_header(msg['Subject'])[0] subjectField = str(decode[0]) if args.verbose: print("[INFO] From: %s Subject: %s" % (fromField, subjectField)) attachments = [] body = '' for part in msg.walk(): if part.get_content_type() == "text/plain": body = part.get_payload(decode=True).decode() else: # Extract MIME parts filename = part.get_filename() mimetype = part.get_content_type() if filename and mimetype: if mimetype in config['caseFiles'] or not config['caseFiles']: print("[INFO] Found attachment: %s (%s)" % (filename, mimetype)) # Decode the attachment and save it in a temporary file charset = part.get_content_charset() if charset is None: charset = chardet.detect(bytes(part))['encoding'] fd, path = tempfile.mkstemp(prefix=slugify(filename) + "_") try: with os.fdopen(fd, 'w+b') as tmp: tmp.write(part.get_payload(decode=1)) attachments.append(path) except OSError as e: print("[ERROR] Cannot dump attachment to %s: %s" % (path, e.errno)) api = TheHiveApi(config['thehiveURL'], config['thehiveUser'], config['thehivePassword'], { 'http': '', 'https': '' }) if '[ALERT]' in subjectField: # Prepare the alert sourceRef = str(uuid.uuid4())[0:6] alert = Alert(title=subjectField.replace('[ALERT]', ''), tlp=int(config['alertTLP']), tags=config['alertTags'], description=body, type='external', source=fromField, sourceRef=sourceRef) # Create the Alert id = None response = api.create_alert(alert) if response.status_code == 201: if args.verbose: print('[INFO] Created alert %s' % response.json()['sourceRef']) else: print('[ERROR] Cannot create alert: %s (%s)' % (response.status_code, response.text)) sys.exit(0) else: # Prepare the sample case tasks = [] for task in config['caseTasks']: tasks.append(CaseTask(title=task)) # Prepare the custom fields customFields = CustomFieldHelper()\ .add_string('from', fromField)\ .add_string('attachment', str(attachments))\ .build() case = Case(title=subjectField, tlp=int(config['caseTLP']), flag=False, tags=config['caseTags'], description=body, tasks=tasks, customFields=customFields) # Create the case id = None response = api.create_case(case) if response.status_code == 201: newID = response.json()['id'] if args.verbose: print('[INFO] Created case %s' % response.json()['caseId']) if len(attachments) > 0: for path in attachments: observable = CaseObservable( dataType='file', data=[path], tlp=int(config['caseTLP']), ioc=False, tags=config['caseTags'], message='Created by imap2thehive.py') response = api.create_case_observable(newID, observable) if response.status_code == 201: if args.verbose: print('[INFO] Added observable %s to case ID %s' % (path, newID)) os.unlink(path) else: print('[ERROR] Cannot add observable: %s - %s (%s)' % (path, response.status_code, response.text)) sys.exit(0) else: print('[ERROR] Cannot create case: %s (%s)' % (response.status_code, response.text)) sys.exit(0) return
def submitTheHive(message): ''' Create a new case in TheHive based on the email Return 'TRUE' is successfully processed otherwise 'FALSE' ''' global log # Decode email msg = email.message_from_bytes(message) decode = email.header.decode_header(msg['From'])[0] if decode[1] is not None: fromField = decode[0].decode(decode[1]) else: fromField = str(decode[0]) decode = email.header.decode_header(msg['Subject'])[0] if decode[1] is not None: subjectField = decode[0].decode(decode[1]) else: subjectField = str(decode[0]) log.info("From: %s Subject: %s" % (fromField, subjectField)) attachments = [] observables = [] # Extract SMTP headers and search for observables parser = HeaderParser() headers = parser.parsestr(msg.as_string()) headers_string = '' i = 0 while i < len(headers.keys()): headers_string = headers_string + headers.keys( )[i] + ': ' + headers.values()[i] + '\n' i += 1 # Temporary disabled # observables = searchObservables(headers_string, observables) body = '' for part in msg.walk(): if part.get_content_type() == "text/plain": try: body = part.get_payload(decode=True).decode() except UnicodeDecodeError: body = part.get_payload(decode=True).decode('ISO-8859-1') observables.extend(searchObservables(body, observables)) elif part.get_content_type() == "text/html": try: html = part.get_payload(decode=True).decode() except UnicodeDecodeError: html = part.get_payload(decode=True).decode('ISO-8859-1') observables.extend(searchObservables(html, observables)) else: # Extract MIME parts filename = part.get_filename() mimetype = part.get_content_type() if filename and mimetype: if mimetype in config['caseFiles'] or not config['caseFiles']: log.info("Found attachment: %s (%s)" % (filename, mimetype)) # Decode the attachment and save it in a temporary file charset = part.get_content_charset() if charset is None: charset = chardet.detect(bytes(part))['encoding'] fd, path = tempfile.mkstemp(prefix=slugify(filename) + "_") try: with os.fdopen(fd, 'w+b') as tmp: tmp.write(part.get_payload(decode=1)) attachments.append(path) except OSerror as e: log.error("Cannot dump attachment to %s: %s" % (path, e.errno)) return False # Cleanup observables (remove duplicates) new_observables = [] for o in observables: if not {'type': o['type'], 'value': o['value']} in new_observables: # Is the observable whitelisted? if isWhitelisted(o['value']): log.debug('Skipping whitelisted observable: %s' % o['value']) else: new_observables.append({ 'type': o['type'], 'value': o['value'] }) log.debug('Found observable %s: %s' % (o['type'], o['value'])) else: log.info('Ignoring duplicate observable: %s' % o['value']) log.info("Removed duplicate observables: %d -> %d" % (len(observables), len(new_observables))) observables = new_observables if config['thehiveAuthMethod'] == 'APIKey': api = TheHiveApi(config['thehiveURL'], config['thehiveAPIKey'], None, { 'http': '', 'https': '' }) else: api = TheHiveApi(config['thehiveURL'], config['thehiveUser'], config['thehivePassword'], { 'http': '', 'https': '' }) # Search for interesting keywords in subjectField: log.debug("Searching for %s in '%s'" % (config['alertKeywords'], subjectField)) if re.match(config['alertKeywords'], subjectField, flags=0): # # Add observables found in the mail body # artifacts = [] if config['thehiveObservables'] and len(observables) > 0: for o in observables: artifacts.append( AlertArtifact(dataType=o['type'], data=o['value'])) # # Prepare tags - add alert keywords found to the list of tags # tags = list(config['alertTags']) match = re.findall(config['alertKeywords'], subjectField) for m in match: tags.append(m) # # Prepare the alert # sourceRef = str(uuid.uuid4())[0:6] alert = Alert(title=subjectField.replace('[ALERT]', ''), tlp=int(config['alertTLP']), tags=tags, description=body, type='external', source=fromField, sourceRef=sourceRef, artifacts=artifacts) # Create the Alert id = None response = api.create_alert(alert) if response.status_code == 201: log.info('Created alert %s' % response.json()['sourceRef']) else: log.error('Cannot create alert: %s (%s)' % (response.status_code, response.text)) return False else: # Prepare the sample case tasks = [] for task in config['caseTasks']: tasks.append(CaseTask(title=task)) # Prepare the custom fields customFields = CustomFieldHelper()\ .add_string('from', fromField)\ .add_string('attachment', str(attachments))\ .build() # If a case template is specified, use it instead of the tasks if len(config['caseTemplate']) > 0: case = Case(title=subjectField, tlp=int(config['caseTLP']), flag=False, tags=config['caseTags'], description=body, template=config['caseTemplate'], customFields=customFields) else: