def _create_alert_config(rule, match): context = {'rule': rule, 'match': match} alert_config = { 'artifacts': _create_artifacts(rule, match), 'sourceRef': str(uuid.uuid4())[0:6], 'title': '{rule[name]}'.format(**context) } alert_config.update(rule.get('hive_alert_config', {})) for alert_config_field, alert_config_value in alert_config.items(): if alert_config_field == 'customFields': custom_fields = CustomFieldHelper() for cf_key, cf_value in alert_config_value.items(): try: func = getattr(custom_fields, 'add_{}'.format(cf_value['type'])) except AttributeError: raise Exception('unsupported custom field type {}'.format(cf_value['type'])) value = cf_value['value'].format(**context) func(cf_key, value) alert_config[alert_config_field] = custom_fields.build() elif isinstance(alert_config_value, str): alert_config[alert_config_field] = alert_config_value.format(**context) elif isinstance(alert_config_value, (list, tuple)): formatted_list = [] for element in alert_config_value: try: formatted_list.append(element.format(**context)) except (AttributeError, KeyError, IndexError): formatted_list.append(element) alert_config[alert_config_field] = formatted_list return alert_config
def alert(self, matches): connection_details = self.rule['hive_connection'] api = TheHiveApi( '{hive_host}:{hive_port}'.format(**connection_details), connection_details.get('hive_apikey',''), proxies=connection_details.get('hive_proxies', {'http': '', 'https': ''}), cert=connection_details.get('hive_verify', False)) for match in matches: context = {'rule': self.rule, 'match': match} artifacts = [] for mapping in self.rule.get('hive_observable_data_mapping', []): for observable_type, match_data_key in mapping.iteritems(): try: match_data_keys = re.findall(r'\{match\[([^\]]*)\]', match_data_key) if all([True for k in match_data_keys if k in context['match']]): artifacts.append(AlertArtifact(dataType=observable_type, data=match_data_key.format(**context))) except KeyError: raise KeyError('\nformat string\n{}\nmatch data\n{}'.format(match_data_key, context)) alert_config = { 'artifacts': artifacts, 'sourceRef': str(uuid.uuid4())[0:6], 'title': '{rule[name]}'.format(**context) } alert_config.update(self.rule.get('hive_alert_config', {})) for alert_config_field, alert_config_value in alert_config.iteritems(): if alert_config_field == 'customFields': custom_fields = CustomFieldHelper() for cf_key, cf_value in alert_config_value.iteritems(): try: func = getattr(custom_fields, 'add_{}'.format(cf_value['type'])) except AttributeError: raise Exception('unsupported custom field type {}'.format(cf_value['type'])) value = cf_value['value'].format(**context) func(cf_key, value) alert_config[alert_config_field] = custom_fields.build() elif isinstance(alert_config_value, basestring): alert_config[alert_config_field] = alert_config_value.format(**context) elif isinstance(alert_config_value, (list, tuple)): formatted_list = [] for element in alert_config_value: try: formatted_list.append(element.format(**context)) except: formatted_list.append(element) alert_config[alert_config_field] = formatted_list alert = Alert(**alert_config) response = api.create_alert(alert) if response.status_code != 201: raise Exception('alert not successfully created in TheHive\n{}'.format(response.text))
def create_alarm(self, title, source_ref=None, description='N/A', alert_type='external', source='LogRhythm', iocs=None, additional_fields=None, additional_tags=None, tlp=TLP.AMBER, pap=PAP.AMBER, severity=HiveSeverity.MEDIUM): if source_ref is None: source_ref = str(uuid.uuid4())[0:6] alert_tags = self.alert_tags.copy() if additional_tags is not None: for additional_tag in additional_tags: alert_tags.append(additional_tag) custom_fields_helper = CustomFieldHelper() if additional_fields is not None: for field in additional_fields: custom_fields_helper.add_string(field['name'], field['value']) custom_fields = custom_fields_helper.build() artifacts = list() if iocs is not None: for ioc in iocs: artifacts.append( AlertArtifact(dataType=ioc['type'].value, data=ioc['value'])) hive_alert = Alert(title=title, tlp=tlp.value, tags=alert_tags, description=description, type=alert_type, source=source, sourceRef=source_ref, pap=pap.value, artifacts=artifacts, customFields=custom_fields, severity=severity.value) response = self.api.create_alert(hive_alert) if response.status_code == 201: print('Alerta Creada Exitosamente') print(json.dumps(response.json(), indent=4, sort_keys=True)) else: print('Error') print(response.text) return response.json()
def create_case_from_inc_name(api, incident): # Prepare the custom fields customFields = CustomFieldHelper() \ .add_string('category', incident["category"]) \ .add_string('type', incident["type"]) \ .build() # Словарь соответствия Severity в MP SIEM и The HIVE severity_dict = {'Low': 1, 'Medium': 2, 'High': 3} case = Case(title=incident["name"], tlp=3, flag=True, tags=['MP SIEM', 'Script'], description='', severity=severity_dict[incident["severity"]], customFields=customFields) # Create the CASE response = api.create_case(case) if response.status_code == 201: # print(json.dumps(response.json(), indent=4, sort_keys=True)) case_id = response.json()['id'] print_log("INFO: New Case **" + incident["name"] + "** in The Hive created") else: print_log('ERROR: {}/{}'.format(response.status_code, response.text)) sys.exit(0) return case_id
def CreateHiveAlertFromSentinel(api, title, description, incidentnumber, severity, source, artifacts, alertIds, incidentURL): tags = [] if severity == "Low": theHiveSeverity = 1 elif severity == "Informational": theHiveSeverity = 1 elif severity == "Medium": theHiveSeverity = 2 elif severity == "High": theHiveSeverity = 3 alertIdsStr = ' '.join(map(str, alertIds)) customFields = CustomFieldHelper() customFields.add_number('sentinelIncidentNumber', incidentnumber) customFields.add_string('alertIds', alertIdsStr) customFields.add_string('incidentURL', incidentURL) customFields = customFields.build() alert = Alert(title=title, tlp=2, tags=tags, description=description, type='Sentinel', severity=theHiveSeverity, source='Sentinel:' + source, customFields=customFields, sourceRef="Sentinel" + str(incidentnumber), artifacts=artifacts) # Create the Alert response = api.create_alert(alert) if response.status_code == 201: logging.info( 'Alert created: ' + 'Sentinel' + str(incidentnumber) + ': ' + source + ': ' + title + '. StatusCode: {}/{}'.format(response.status_code, response.text)) elif (response.status_code == 400 and response.json()['type'] == "ConflictError"): logging.info( 'Duplicate alert: ' + 'Sentinel' + str(incidentnumber) + ': ' + source + ': ' + title + '. StatusCode: {}/{}'.format(response.status_code, response.text)) else: logging.error( 'failed to create alert: ' + source + ' ' + title + ' Sentinel' + str(incidentnumber) + '. StatusCode: {}/{}'.format(response.status_code, response.text)) sys.exit(0)
def create_case(self, title, tasks=None, tlp=TLP.AMBER, pap=PAP.AMBER, severity=HiveSeverity.MEDIUM, additional_fields=None, additional_tags=None, flag=False, description='N/A'): case_tags = self.case_tags.copy() if additional_tags is not None: for additional_tag in additional_tags: case_tags.append(additional_tag) custom_fields_helper = CustomFieldHelper() if additional_fields is not None: for field in additional_fields: custom_fields_helper.add_string(field['name'], field['value']) custom_fields = custom_fields_helper.build() new_tasks = list() if tasks is not None: for task in tasks: new_tasks.append(CaseTask(title=task)) hive_case = Case(title=title, tlp=tlp.value, pap=pap.value, description=description, tags=case_tags, severity=severity.value, flag=flag, customFields=custom_fields, tasks=new_tasks) response = self.api.create_case(hive_case) if response.status_code == 201: print('Caso Creada Exitosamente') print(json.dumps(response.json(), indent=4, sort_keys=True)) else: print('Error') print(response.text) return response.json()
def test_intger(): custom_fields = CustomFieldHelper() \ .add_float('float-field', 1.0) \ .build() assert custom_fields is not None assert 'float-field' in custom_fields assert 'float' in custom_fields['float-field'] assert 'order' in custom_fields['float-field'] assert custom_fields['float-field']['float'] == 1.0 assert custom_fields['float-field']['order'] == 0
def test_intger(): custom_fields = CustomFieldHelper() \ .add_integer('integer-field', 100) \ .build() assert custom_fields is not None assert 'integer-field' in custom_fields assert 'integer' in custom_fields['integer-field'] assert 'order' in custom_fields['integer-field'] assert custom_fields['integer-field']['integer'] == 100 assert custom_fields['integer-field']['order'] == 0
def test_boolean(): custom_fields = CustomFieldHelper() \ .add_boolean('boolean-field', True) \ .build() assert custom_fields is not None assert 'boolean-field' in custom_fields assert 'boolean' in custom_fields['boolean-field'] assert 'order' in custom_fields['boolean-field'] assert custom_fields['boolean-field']['boolean'] is True assert custom_fields['boolean-field']['order'] == 0
def test_string(): custom_fields = CustomFieldHelper() \ .add_string('string-field', 'thehive') \ .build() assert custom_fields is not None assert 'string-field' in custom_fields assert 'string' in custom_fields['string-field'] assert 'order' in custom_fields['string-field'] assert custom_fields['string-field']['string'] == 'thehive' assert custom_fields['string-field']['order'] == 0
def test_date(): date = int(time.time()) * 1000 custom_fields = CustomFieldHelper() \ .add_date('date-field', date) \ .build() assert custom_fields is not None assert 'date-field' in custom_fields assert 'date' in custom_fields['date-field'] assert 'order' in custom_fields['date-field'] assert custom_fields['date-field']['date'] == date assert custom_fields['date-field']['order'] == 0
def __init__(self, event, tenant): # Case attributes user = event['user'] nature = event['nature'] case_name = event['case']['name'] case_id = event['case']['id'] case_url = event['case']['url'] case_tags = ['McAfee Investigator', case_id, nature] # Task attributes task_owner = tenant.hive_user # task_start_date = datetime.datetime.now().strftime("%d-%m-%Y %H:%M") task_start_date = int(time.time()) * 1000 task_description = "Autogenerated task to track MI case events" tasks = [ CaseTask(title='MI - analytics investigation', status='InProgress', owner=task_owner, startDate=task_start_date, description=task_description), CaseTask(title='MI - system events', status='InProgress', owner=task_owner, startDate=task_start_date, description=task_description), CaseTask(title='MI - user events', status='Waiting', description="Auto generated task to track MI user events") ] # Prepare the custom fields (values are searchable) customFields = CustomFieldHelper()\ .add_boolean('MI-autogenerated', True)\ .add_string('MI-caseid', case_id)\ .add_string('MI-user', user)\ .add_string('MI-nature', nature)\ .build() case = Case(title=case_name, tlp=0, flag=False, description="For additional info see " + self.get_case_ui_url(case_url), tags=case_tags, tasks=tasks, customFields=customFields) self.case = case
def sendtoHIVE(title, description, domain): tasks = [ CaseTask(title='Tracking'), CaseTask(title='Communication'), CaseTask(title='Investigation', status='Waiting', flag=True) ] # Prepare the custom fields customFields = CustomFieldHelper()\ .add_boolean('booleanField', True)\ .add_string('businessImpact', 'HIGH')\ .add_date('occurDate', int(time.time())*1000)\ .add_number('cvss', 9)\ .build() case = Case(title=title, tlp=3, flag=True, tags=['wordpress', domain], description=description, tasks=tasks, customFields=customFields) # Create the case print('Create Case') print('-----------------------------') id = None response = api.create_case(case) if response.status_code == 201: print(json.dumps(response.json(), indent=4, sort_keys=True)) print('') id = response.json()['id'] else: print('ko: {}/{}'.format(response.status_code, response.text)) sys.exit(0) # Get all the details of the created case print('Get created case {}'.format(id)) print('-----------------------------') response = api.get_case(id) if response.status_code == requests.codes.ok: print(json.dumps(response.json(), indent=4, sort_keys=True)) print('') else: print('ko: {}/{}'.format(response.status_code, response.text))
def parse_hooks(self): logger.info('Start parsing webhooks for ELK automation') #Set the default action self.report_action = 'None' #if it is a new case if self.webhook.isImportedAlert(): logger.info('Registered the finishing steps of the creation of a new case') #Register case info self.caseid = self.webhook.data['details']['case'] self.tags = self.webhook.data['object']['tags'] self.description = self.webhook.data['object']['description'] #Enrich the case information with missing information from the alert if 'ELK' in self.webhook.data['object']['tags']: self.report_action = 'enrichCase' logger.info('Alert {} has been tagged as ELK and is just imported or following has been reenabled. Adding custom fields'.format(self.webhook.data['details']['case'])) # Enrich offense with information from the alert by posting the missing information through an API call #Create a Case object self.case = Case() #Add the case id to the object self.case.id = self.caseid logger.info('Updating case %s' % self.case.id) #Define which fields need to get updated fields = ['customFields'] #Retrieve all required attributes from the alert and add them as custom fields to the case customFields = CustomFieldHelper()\ .add_string('anomalyType', self.webhook.data['object']['type'])\ .add_string('source', self.webhook.data['object']['source'])\ .build() #Add custom fields to the case object self.case.customFields = customFields #Update the case response = self.TheHiveConnector.updateCase(self.case,fields) return report_action
def test_order(): custom_fields = CustomFieldHelper()\ .add_boolean('booleanField', True)\ .add_string('businessImpact', 'HIGH')\ .add_date('occurDate', int(time.time())*1000)\ .add_number('cvss', 9) \ .add_integer('csirts', 2) \ .add_float('hitRate', 50.2) \ .build() assert custom_fields is not None assert 'booleanField' in custom_fields assert 'businessImpact' in custom_fields assert 'occurDate' in custom_fields assert 'cvss' in custom_fields assert 'csirts' in custom_fields assert 'hitRate' in custom_fields assert custom_fields['booleanField']['order'] == 0 assert custom_fields['businessImpact']['order'] == 1 assert custom_fields['occurDate']['order'] == 2 assert custom_fields['cvss']['order'] == 3 assert custom_fields['csirts']['order'] == 4 assert custom_fields['hitRate']['order'] == 5
def rapid7IDRAlerts2Alerts(alert_data, org_name): logger = logging.getLogger('workflows.' + __name__) logger.info('%s.rapid7IDRAlerts2Alert starts', __name__) result = {} result['success'] = bool() conf = getConf() theHiveConnector = TheHiveConnector(conf) logger.info("Building custom fields ...") customFields = CustomFieldHelper()\ .add_string('client', org_name)\ .build() tags = [] now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M") alert_date = dateutil_parser.parse(alert_data.get('timestamp', now)) for user in alert_data.get('actors', {}).get('users', []): tags.append(user.get('name', "")) for asset in alert_data.get('actors', {}).get('assets', []): tags.append(asset.get('shortname', "")) logger.info("Building description ...") description = descriptionCrafter(alert_data, org_name) logger.info("Building alert ...") alert = theHiveConnector.craftAlert( title=alert_data.get( 'title', alert_data.get('name', "New alert from Rapid7 Insight IDR")), description=description, severity=2, #is there a way to determine efficiently this thing ? date=int(alert_date.timestamp()) * 1000, tags=tags, tlp=2, status="New", type="SIEM", source="Rapid7 Insight IDR", sourceRef=alert_data.get('investigationId', str(uuid.uuid4())), artifacts=artifactsCrafter(alert_data), caseTemplate="Insight IDR Case", customFields=customFields) logger.info("Sending alert to TheHive ...") try: ret = theHiveConnector.createAlert(alert) logger.info("Alert {} created in TheHive".format(str(ret['id']))) result['success'] = True except ValueError: logger.warning("Alert creation failed, trying to update ...") try: ret = theHiveConnector.updateAlert(alert.sourceRef, alert) logger.info("Alert {} updated in TheHive".format(str(ret['id']))) result['success'] = True except Exception as error: logger.error("Alert update failed ! {}".format(error)) result['success'] = False return result
def submitTheHive(message): '''Create a new case in TheHive based on the email''' # Decode email msg = email.message_from_bytes(message) decode = email.header.decode_header(msg['From'])[0] fromField = str(decode[0]) decode = email.header.decode_header(msg['Subject'])[0] subjectField = str(decode[0]) if args.verbose: print("[INFO] From: %s Subject: %s" % (fromField, subjectField)) attachments = [] body = '' for part in msg.walk(): if part.get_content_type() == "text/plain": body = part.get_payload(decode=True).decode() else: # Extract MIME parts filename = part.get_filename() mimetype = part.get_content_type() if filename and mimetype: if mimetype in config['caseFiles'] or not config['caseFiles']: print("[INFO] Found attachment: %s (%s)" % (filename, mimetype)) # Decode the attachment and save it in a temporary file charset = part.get_content_charset() if charset is None: charset = chardet.detect(bytes(part))['encoding'] fd, path = tempfile.mkstemp(prefix=slugify(filename) + "_") try: with os.fdopen(fd, 'w+b') as tmp: tmp.write(part.get_payload(decode=1)) attachments.append(path) except OSError as e: print("[ERROR] Cannot dump attachment to %s: %s" % (path, e.errno)) api = TheHiveApi(config['thehiveURL'], config['thehiveUser'], config['thehivePassword'], { 'http': '', 'https': '' }) if '[ALERT]' in subjectField: # Prepare the alert sourceRef = str(uuid.uuid4())[0:6] alert = Alert(title=subjectField.replace('[ALERT]', ''), tlp=int(config['alertTLP']), tags=config['alertTags'], description=body, type='external', source=fromField, sourceRef=sourceRef) # Create the Alert id = None response = api.create_alert(alert) if response.status_code == 201: if args.verbose: print('[INFO] Created alert %s' % response.json()['sourceRef']) else: print('[ERROR] Cannot create alert: %s (%s)' % (response.status_code, response.text)) sys.exit(0) else: # Prepare the sample case tasks = [] for task in config['caseTasks']: tasks.append(CaseTask(title=task)) # Prepare the custom fields customFields = CustomFieldHelper()\ .add_string('from', fromField)\ .add_string('attachment', str(attachments))\ .build() case = Case(title=subjectField, tlp=int(config['caseTLP']), flag=False, tags=config['caseTags'], description=body, tasks=tasks, customFields=customFields) # Create the case id = None response = api.create_case(case) if response.status_code == 201: newID = response.json()['id'] if args.verbose: print('[INFO] Created case %s' % response.json()['caseId']) if len(attachments) > 0: for path in attachments: observable = CaseObservable( dataType='file', data=[path], tlp=int(config['caseTLP']), ioc=False, tags=config['caseTags'], message='Created by imap2thehive.py') response = api.create_case_observable(newID, observable) if response.status_code == 201: if args.verbose: print('[INFO] Added observable %s to case ID %s' % (path, newID)) os.unlink(path) else: print('[ERROR] Cannot add observable: %s - %s (%s)' % (path, response.status_code, response.text)) sys.exit(0) else: print('[ERROR] Cannot create case: %s (%s)' % (response.status_code, response.text)) sys.exit(0) return
def submitTheHive(message): ''' Create a new case in TheHive based on the email Return 'TRUE' is successfully processed otherwise 'FALSE' ''' global log # Decode email msg = email.message_from_bytes(message) decode = email.header.decode_header(msg['From'])[0] if decode[1] is not None: fromField = decode[0].decode(decode[1]) else: fromField = str(decode[0]) decode = email.header.decode_header(msg['Subject'])[0] if decode[1] is not None: subjectField = decode[0].decode(decode[1]) else: subjectField = str(decode[0]) log.info("From: %s Subject: %s" % (fromField, subjectField)) attachments = [] observables = [] # Extract SMTP headers and search for observables parser = HeaderParser() headers = parser.parsestr(msg.as_string()) headers_string = '' i = 0 while i < len(headers.keys()): headers_string = headers_string + headers.keys( )[i] + ': ' + headers.values()[i] + '\n' i += 1 # Temporary disabled # observables = searchObservables(headers_string, observables) body = '' for part in msg.walk(): if part.get_content_type() == "text/plain": try: body = part.get_payload(decode=True).decode() except UnicodeDecodeError: body = part.get_payload(decode=True).decode('ISO-8859-1') observables.extend(searchObservables(body, observables)) elif part.get_content_type() == "text/html": try: html = part.get_payload(decode=True).decode() except UnicodeDecodeError: html = part.get_payload(decode=True).decode('ISO-8859-1') observables.extend(searchObservables(html, observables)) else: # Extract MIME parts filename = part.get_filename() mimetype = part.get_content_type() if filename and mimetype: if mimetype in config['caseFiles'] or not config['caseFiles']: log.info("Found attachment: %s (%s)" % (filename, mimetype)) # Decode the attachment and save it in a temporary file charset = part.get_content_charset() if charset is None: charset = chardet.detect(bytes(part))['encoding'] fd, path = tempfile.mkstemp(prefix=slugify(filename) + "_") try: with os.fdopen(fd, 'w+b') as tmp: tmp.write(part.get_payload(decode=1)) attachments.append(path) except OSerror as e: log.error("Cannot dump attachment to %s: %s" % (path, e.errno)) return False # Cleanup observables (remove duplicates) new_observables = [] for o in observables: if not {'type': o['type'], 'value': o['value']} in new_observables: # Is the observable whitelisted? if isWhitelisted(o['value']): log.debug('Skipping whitelisted observable: %s' % o['value']) else: new_observables.append({ 'type': o['type'], 'value': o['value'] }) log.debug('Found observable %s: %s' % (o['type'], o['value'])) else: log.info('Ignoring duplicate observable: %s' % o['value']) log.info("Removed duplicate observables: %d -> %d" % (len(observables), len(new_observables))) observables = new_observables if config['thehiveAuthMethod'] == 'APIKey': api = TheHiveApi(config['thehiveURL'], config['thehiveAPIKey'], None, { 'http': '', 'https': '' }) else: api = TheHiveApi(config['thehiveURL'], config['thehiveUser'], config['thehivePassword'], { 'http': '', 'https': '' }) # Search for interesting keywords in subjectField: log.debug("Searching for %s in '%s'" % (config['alertKeywords'], subjectField)) if re.match(config['alertKeywords'], subjectField, flags=0): # # Add observables found in the mail body # artifacts = [] if config['thehiveObservables'] and len(observables) > 0: for o in observables: artifacts.append( AlertArtifact(dataType=o['type'], data=o['value'])) # # Prepare tags - add alert keywords found to the list of tags # tags = list(config['alertTags']) match = re.findall(config['alertKeywords'], subjectField) for m in match: tags.append(m) # # Prepare the alert # sourceRef = str(uuid.uuid4())[0:6] alert = Alert(title=subjectField.replace('[ALERT]', ''), tlp=int(config['alertTLP']), tags=tags, description=body, type='external', source=fromField, sourceRef=sourceRef, artifacts=artifacts) # Create the Alert id = None response = api.create_alert(alert) if response.status_code == 201: log.info('Created alert %s' % response.json()['sourceRef']) else: log.error('Cannot create alert: %s (%s)' % (response.status_code, response.text)) return False else: # Prepare the sample case tasks = [] for task in config['caseTasks']: tasks.append(CaseTask(title=task)) # Prepare the custom fields customFields = CustomFieldHelper()\ .add_string('from', fromField)\ .add_string('attachment', str(attachments))\ .build() # If a case template is specified, use it instead of the tasks if len(config['caseTemplate']) > 0: case = Case(title=subjectField, tlp=int(config['caseTLP']), flag=False, tags=config['caseTags'], description=body, template=config['caseTemplate'], customFields=customFields) else:
def connectEws(): report = dict() report['success'] = bool() clients = None with open('/opt/Synapse/conf/mailboxes.json') as f: clients = json.load(f) mailboxes = clients['mailboxes'] cfg = getConf() theHiveConnector = TheHiveConnector(cfg) for mailbox in mailboxes: ewsConnector = EwsConnector(username=mailbox['username'], password=mailbox['password'], auth_type=mailbox['auth_type'], server=mailbox['server'], smtp_address=mailbox['smtp_address']) unread = ewsConnector.scan(mailbox['folder_name']) for msg in unread: #type(msg) #<class 'exchangelib.folders.Message'> conversationId = msg.conversation_id.id origin_mail = msg.attachments[0].item fullBody = getEmailBody(msg) alertTitle = "{}; {}".format( msg.subject, ''.join( [i if ord(i) < 128 else '' for i in origin_mail.subject])) alertDescription = ('```\n' + 'Alert created by Synapse\n' + 'conversation_id: "' + str(msg.conversation_id.id) + '"\n\n' + 'Original mail:\n\n' + fullBody[3:]) # rapporteur reporter = str(msg.sender.email_address).lower() # expéditeur sender = str(origin_mail.sender.email_address).lower() # date de réception originale et en-tant que spam rcv_date_timestamp = origin_mail.datetime_received.timestamp( ) * 1000 # date de signalement à la boîte support rpt_date_timestamp = msg.datetime_received.timestamp() * 1000 customFields = CustomFieldHelper()\ .add_string('reporter', reporter)\ .add_string('sender', sender)\ .add_date('receivedDate', rcv_date_timestamp)\ .add_date('reportedDate', rpt_date_timestamp)\ .add_string('client', mailbox['name'])\ .build() file_paths = [] artifacts = [] artifacts.append( AlertArtifact(dataType='mail', data=reporter, ignoreSimilarity=True, tags=['Synapse'])) artifacts.append( AlertArtifact(dataType='mail', data=sender, message="Original sender of the e-mail", tags=['Synapse', 'Sender'])) artifacts.append( AlertArtifact(dataType='mail_subject', data=str(origin_mail.subject), message="Original subject of the e-mail", tags=['Synapse'])) attachedFiles = getFileAttachments(msg) for attached in attachedFiles: artifacts.append( AlertArtifact(dataType='file', data=[attached['path']], tags=['Attachment', 'Synapse'])) file_paths.append(attached['path']) alert = Alert(title=alertTitle, description=alertDescription, severity=2, date=rcv_date_timestamp, tags=[], tlp=2, status="New", type="Phishing", source="BAL Phishing", sourceRef="{}-{}".format( mailbox['short_name'], str(uuid.uuid4())[24:].upper()), artifacts=artifacts, caseTemplate="Suspicious Email Case", customFields=customFields) theHiveConnector.createAlert(alert) ewsConnector.markAsRead(msg) logger.info("Cleaning temp files ...") for temp_file in file_paths: try: os.remove(temp_file) except OSError as errRm: logger.error("'{}' could not be removed.{}".format( temp_file, errRm)) report['success'] = True return report
def allOffense2Alert(timerange): """ Get all openned offense created within the last <timerange> minutes and creates alerts for them in TheHive """ logger = logging.getLogger(__name__) logger.info('%s.allOffense2Alert starts', __name__) report = dict() report['success'] = True report['offenses'] = list() try: cfg = getConf() qradarConnector = QRadarConnector(cfg) theHiveConnector = TheHiveConnector(cfg) offensesList = qradarConnector.getOffenses(timerange) #each offenses in the list is represented as a dict #we enrich this dict with additional details for offense in offensesList: #searching if the offense has already been converted to alert logger.info('Looking for offense %s in TheHive alerts', str(offense['id'])) # Update only new Alerts, as Ignored it will be closed on QRadar and should not be updated, # as Imported we will do a responder to fetch latest info in the case results = theHiveConnector.findAlert( Eq("sourceRef", str(offense['id']))) offense_report = dict() try: if len(results) == 0: logger.info( 'Offense %s not found in TheHive alerts, creating it', str(offense['id'])) enrichedOffense = enrichOffense(qradarConnector, offense) theHiveAlert = qradarOffenseToHiveAlert( theHiveConnector, enrichedOffense) theHiveEsAlertId = theHiveConnector.createAlert( theHiveAlert)['id'] offense_report['type'] = "Creation" offense_report['raised_alert_id'] = theHiveEsAlertId offense_report['qradar_offense_id'] = offense['id'] offense_report['success'] = True report['offenses'].append(offense_report) elif results[0]['status'] not in ['Ignored', 'Imported']: # update alert if alert is not imported and not dimissed # will only update 'lastEventCount' and 'lastUpdatedTime' custom fields logger.info('Updating offense %s', str(offense['id'])) alert = Alert(json=results[0]) cf = CustomFieldHelper() alert.title = offense['description'] if 'lastEventCount' not in alert.customFields: alert.customFields['lastEventCount'] = {} if 'lastUpdated' not in alert.customFields: alert.customFields['lastUpdated'] = {} if 'offenseSource' not in alert.customFields: alert.customFields['offenseSource'] = {} alert.customFields['lastEventCount']['number'] = offense[ 'event_count'] alert.customFields['lastUpdated']['date'] = offense[ 'last_updated_time'] alert.customFields['offenseSource']['string'] = offense[ 'offense_source'] # updated maybe ? # should improve TheHiveConnector.updateAlert() rather than using this updatedAlert = theHiveConnector.theHiveApi.update_alert( results[0]['id'], alert, fields=['customFields', 'title']) if not updatedAlert.ok: raise ValueError(json.dumps(updatedAlert.json())) offense_report['type'] = "Update" offense_report['updated_alert_id'] = updatedAlert.json( )['id'] offense_report['qradar_offense_id'] = offense['id'] offense_report['success'] = True report['offenses'].append(offense_report) else: logger.info("Offense already exists") except Exception as e: logger.error('%s.allOffense2Alert failed', __name__, exc_info=True) offense_report['success'] = False if isinstance(e, ValueError): errorMessage = json.loads(str(e))['message'] offense_report['message'] = errorMessage else: offense_report['message'] = str( e) + ": Couldn't raise alert in TheHive" offense_report['offense_id'] = offense['id'] # Set overall success if any fails report['success'] = False except Exception as e: logger.error( 'Failed to create alert from QRadar offense (retrieving offenses failed)', exc_info=True) report['success'] = False report['message'] = "%s: Failed to create alert from offense" % str(e) return report
sl.insert_record( enrichment_table, 'id, enrichment_id, enrichment_type, status', '"{}","{}","{}","{}"'.format(offense_id, aql_id, 'username', 'Open')) records = qr.get_aql_results(aql_id) if records: retval = parse_get_aql(records, 'username') if retval: for item in retval: if item != None: username_list.append(item) sl.update_record(enrichment_table, 'status', 'Closed', 'enrichment_id', aql_id) # # Adding custom_fields values to the case - static mapping. custom_fields = CustomFieldHelper() custom_fields.add_number('qradar_id', offense_id) custom_fields.add_string('offense_source', offense_source) custom_fields.build() tlp = offense_severity_mapper(offense_magnitude)['sev'] # #Case - Offense summary md. build_desc = """|Offense Summary:|\n|---|\n|Offense Description: {}|\n|Source NW: {}|\n|Destination NW: {}|\n|Source IPs: {}|\n|Local Destination IPs: {}|\n|Remote Destination IPs: {}|\n|Usernames: {}|\n---\nLink to the Offense: {}""".format( offense_desc, offense_src_nw, offense_dst_nw, source_address_list, local_dest_list, remote_dest_list, username_list, offense_link) # #Some sample tasks-response actions for posting in the case. Customize per your reqs. # #TODO: You can also utilize - thehive-playbook-creator - for dynamic task/playbook assignment using your QRadar Rule groups. tasks = [ CaseTask(title='PB:- Phase:Identification'),
api = TheHiveApi('http://127.0.0.1:9000', '**YOUR_API_KEY**') # Prepare the sample case tasks = [ CaseTask(title='Identify'), CaseTask(title='Contain'), CaseTask(title='Investigate'), CaseTask(title='Eradicate'), CaseTask(title='Recovery'), CaseTask(title='Lessons Learned', status='Waiting', flag=True) ] # Prepare the custom fields customFields = CustomFieldHelper()\ .add_boolean('booleanField', True)\ .add_string('businessImpact', 'HIGH')\ .add_date('occurDate', int(time.time())*1000)\ .add_number('cvss', 9)\ .build() case = Case(title='Account Compromise', tlp=3, flag=True, tags=['TheHive4Py', 'sample'], description='N/A', tasks=tasks, customFields=customFields) # Create the case print('Create Case') print('-----------------------------') id = None
def updateACase(caseId, bigGroup, fromField, attachments): api = TheHiveApi(config['thehiveURL'], config['thehiveUser'], config['thehivePassword'], { 'http': '', 'https': '' }) resolved = parseBody("Resolved", bigGroup) resolved = resolved.lower() Tags = parseBody("Tags", bigGroup) Title = parseBody("Title", bigGroup) TLP = parseBody("TLP", bigGroup) Description = parseBody("Description", bigGroup) AlbertID = parseBody("Albert Id", bigGroup) Severity = parseBody("Severity", bigGroup) ReplaceTags = parseBody("ReplaceTags", bigGroup) ReplaceTags = ReplaceTags.lower() resolvedStatus = parseBody( "Resolution Status", bigGroup ) # (Indeterminate,FalsePositive, TruePositive, Other or Duplicated) ImpactStatus = parseBody( "Impact Status", bigGroup) #(NoImpact, WithImpact or NotApplicable) - Summary = parseBody("Summary", bigGroup) query = Eq('caseId', caseId) try: d = api.find_first(query=query) for key, value in d.items(): if key == "_routing": requestCase = value except: print( "unable to find the caseId in the update email, this is required to uniquely identify a case to update. ending.." ) sys.exit(0) try: updated_case = api.case.update(requestCase) print(updated_case.jsonify()) updated_case.title = Title updated_case.description = Description try: TLP = int(TLP) updated_case.tlp = TLP except: print("Issue with updating TLP, skipping") try: Severity = int(Severity) updated_case.severity = Severity except: print("Issue with updating severity, skipping") print("TAGS HERE") print(Tags) if len(Tags) > 1: newTags = Tags.split(",") if "no" in ReplaceTags: #if replaceTags is no, don't replace tags, instead, append the unique ones for x in newTags: if x not in updated_case.tags: if x + "\r" in updated_case.tags: pass else: updated_case.tags.append(x) #append the tag else: #if replacetags is yes or anything else, replace all tags with specified print("replacing tags") updated_case.tags = newTags # (Indeterminate,FalsePositive, TruePositive, Other or Duplicated) correctResolutions = [ "Indeterminate", "FalsePositive", "TruePositive", "Other", "Duplicated" ] if resolvedStatus not in correctResolutions: print("invalid resolution status") pass if resolvedStatus in correctResolutions: updated_case.resolutionStatus = resolvedStatus correctImpactStatus = ["NoImpact", "WithImpact", "NotApplicable"] if ImpactStatus not in correctImpactStatus: print("invalid impact status") pass if ImpactStatus in correctImpactStatus: updated_case.impactStatus = ImpactStatus updated_case.summary = Summary #overwrite albert id if len(AlbertID) > 1: customFields = CustomFieldHelper() \ .add_string('from', fromField) \ .add_string('attachment', str(attachments)) \ .add_string('albertId', AlbertID) \ .build() updated_case.customFields = customFields if "yes" in resolved: print("resolved") resolveUpdate = api.case.update(requestCase, status='Resolved') if "no" in resolved: print("OPEN") resolveUpdate = api.case.update(requestCase, status='Open') api.update_case(updated_case) except FileExistsError as e: print("Error updating case. {}".format(e))
def submitTheHive(message): ''' Create a new case in TheHive based on the email Return 'TRUE' is successfully processed otherwise 'FALSE' ''' # Decode email msg = email.message_from_bytes(message) # gets full content of the email decode = email.header.decode_header(msg['From'])[0] fromField = str(decode[0]) decode = email.header.decode_header(msg['Subject'])[0] subjectField = str(decode[0]) if args.verbose: print("[INFO] From: %s Subject: %s" % (fromField, subjectField)) attachments = [] observables = [] body = '' bodyMessage = '' for part in msg.walk(): if part.get_content_type() == "text/plain": body = part.get_payload(decode=True).decode() bodyMessage += body observables = searchObservables( body, observables ) # searches the body of the email for supplied observables elif part.get_content_type( ) == "text/html": # if email is html based will search throuh html source code if args.verbose: print("[INFO] Searching for observable in HTML code") html = part.get_payload(decode=True).decode() observables = searchObservables(html, observables) elif part.get_content_type( ) == "application/vnd.ms-excel": #ONLY WORKS FOR .CSV body = part.get_payload(decode=True).decode('UTF-8') observables = searchObservables(body, observables) else: # Extract MIME parts filename = part.get_filename() mimetype = part.get_content_type() if filename and mimetype: if mimetype in config['caseFiles'] or not config['caseFiles']: print("[INFO] Found attachment: %s (%s)" % (filename, mimetype)) # Decode the attachment and save it in a temporary file charset = part.get_content_charset() if charset is None: charset = chardet.detect(bytes(part))['encoding'] fd, path = tempfile.mkstemp(prefix=slugify(filename) + "_") try: with os.fdopen(fd, 'w+b') as tmp: tmp.write(part.get_payload(decode=1)) attachments.append(path) except OSError as e: print("[ERROR] Cannot dump attachment to %s: %s" % (path, e.errno)) return False api = TheHiveApi(config['thehiveURL'], config['thehiveUser'], config['thehivePassword'], { 'http': '', 'https': '' }) # if '[ALERT]' in subjectField: if re.match(config['alertKeywords'], subjectField, flags=0): # # Add observables found in the mail body # artifacts = [] if config['thehiveObservables'] and len(observables) > 0: print("t1") for o in observables: print("t2") artifacts.append( AlertArtifact(dataType=o['type'], data=o['value'])) # # Prepare tags - add alert keywords found to the list of tags # tags = config['alertTags'] match = re.findall(config['alertKeywords'], subjectField) for m in match: tags.append(m) # # Prepare the alert # sourceRef = str(uuid.uuid4())[0:6] alert = Alert( title=subjectField.replace('[ALERT]', ''), tlp=int( config['alertTLP'] ), #setting it blank since custom template allows default color, set it back to tlp = int for conf value tags=tags, description=body, type='external', source=fromField, sourceRef=sourceRef, artifacts=artifacts) # Create the Alert id = None response = api.create_alert(alert) if response.status_code == 201: if args.verbose: print('[INFO] Created alert %s' % response.json()['sourceRef']) else: print('[ERROR] Cannot create alert: %s (%s)' % (response.status_code, response.text)) return False else: # Prepare the sample case tasks = [] for task in config['caseTasks']: tasks.append(CaseTask(title=task)) # Prepare the custom fields customFields = CustomFieldHelper() \ .add_string('from', fromField) \ .add_string('attachment', str(attachments)) \ .build() # If a case template is specified, use it instead of the tasks m = 1 if m == 1: templates = [] for task in config['caseTemplates']: templates.append(task) temptouse = config['caseTemplate'] descrip = re.compile('-"(.+)"') name = re.compile('(.+)-"') for x in templates: z = descrip.search(x) tempVar = name.search(x) searchVar = z.group(1) tempVar = tempVar.group(1) if searchVar in subjectField: print( x ) #if 2 template names in subject, take the latest defined temptouse = tempVar print("TEMPLATE", temptouse) if body: testerVar = False print("body") try: albert = re.compile('Albert Incident #: (\d+)') m = albert.search(body) albertId = m.group(1) print(albertId) customFields = CustomFieldHelper() \ .add_string('from', fromField) \ .add_string('attachment', str(attachments)) \ .add_string('albertId', albertId) \ .build() print(customFields) except: print("albert id doesnt exist") if "Update" in subjectField: #update code testerVar = True print("UPDATE") #INTIAL try: findBodyInfo = re.compile('---((?:.+[\r\n]+)+)---') except: print( "Unable to update, unable to find two '---'s, exiting.." ) sys.exit(0) m = findBodyInfo.search(body) bigGroup = m.group(1) caseId = parseBody("Case Id", bigGroup) print("caseid", caseId) try: caseId = int(caseId) except: print("invalid case id") updateACase(caseId, bigGroup, fromField, attachments) id = None if testerVar == True: print("g") sys.exit(0) #end update code caseTags = [] for tag in config['caseTags']: descripFound = descrip.search(tag) nameFound = name.search(tag) descripFound = descripFound.group(1) nameFound = nameFound.group(1) if descripFound == 'always': caseTags.append(nameFound) elif descripFound in bodyMessage: caseTags.append(nameFound) try: # # Add observables found in the mail body # artifacts = [] if config['thehiveObservables'] and len(observables) > 0: for o in observables: artifacts.append( AlertArtifact(dataType=o['type'], data=o['value'])) # # Prepare tags - add alert keywords found to the list of tags # tags = config['alertTags'] match = re.findall(config['alertKeywords'], subjectField) for m in match: tags.append(m) # # Prepare the alert # sourceRef = str(uuid.uuid4())[0:6] alert = Alert( title=subjectField, tlp=int( config['alertTLP'] ), #setting it blank since custom template allows default color, set it back to tlp = int for conf value tags=caseTags, description=body, type='external', source=fromField, sourceRef=sourceRef, customFields=customFields, severity=None, artifacts=artifacts) except FileExistsError: print( "Error with creating alert, wrong template name or tags?") else: print("") # Create the alert response = api.create_alert(alert) print("Alert being created..") if response.status_code == 201: newID = response.json()['id'] if args.verbose: print('[INFO] Created alert %s' % response.json()['caseId']) if len(attachments) > 0: for path in attachments: observable = CaseObservable( dataType='file', data=[path], tlp=int(config['caseTLP']), ioc=False, tags=config['caseTags'], message='Found as email attachment') response = api.create_case_observable(newID, observable) if response.status_code == 201: if args.verbose: print('[INFO] Added observable %s to case ID %s' % (path, newID)) os.unlink(path) else: print('[WARNING] Cannot add observable: %s - %s (%s)' % (path, response.status_code, response.text)) # # Add observables found in the mail body # if config['thehiveObservables'] and len(observables) > 0: for o in observables: observable = CaseObservable( dataType=o['type'], data=o['value'], tlp=int(config['caseTLP']), ioc=False, tags=caseTags, #switched to custom tags message='Found in the email body') response = api.create_case_observable(newID, observable) if response.status_code == 201: if args.verbose: print( '[INFO] Added observable %s: %s to case ID %s' % (o['type'], o['value'], newID)) else: print( '[WARNING] Cannot add observable %s: %s - %s (%s)' % (o['type'], o['value'], response.status_code, response.text)) else: print('[ERROR] Cannot create case: %s (%s)' % (response.status_code, response.text)) return False return True
def submitTheHive(message): '''Create a new case in TheHive based on the email''' # Decode email msg = email.message_from_bytes(message) decode = email.header.decode_header(msg['From'])[0] fromField = str(decode[0]) decode = email.header.decode_header(msg['Subject'])[0] subjectField = str(decode[0]) if args.verbose: print("[INFO] From: %s Subject: %s" % (fromField, subjectField)) attachments = [] body = '' for part in msg.walk(): if part.get_content_type() == "text/plain": body = part.get_payload(decode=True).decode() else: filename = part.get_filename() if filename: print("[INFO] Found attachment: %s" % filename) attachments.append(filename) api = TheHiveApi(config['thehiveURL'], config['thehiveUser'], config['thehivePassword'], { 'http': '', 'https': '' }) if '[ALERT]' in subjectField: # Prepare the alert sourceRef = str(uuid.uuid4())[0:6] alert = Alert(title=subjectField.replace('[ALERT]', ''), tlp=int(config['alertTLP']), tags=config['alertTags'], description=body, type='external', source=fromField, sourceRef=sourceRef) # Create the Alert id = None response = api.create_alert(alert) if response.status_code == 201: if args.verbose: print('[INFO] Created alert %s' % response.json()['sourceRef']) else: print('[ERROR] Cannot create alert: %s (%s)' % (response.status_code, response.text)) sys.exit(0) else: # Prepare the sample case tasks = [] for task in config['caseTasks']: tasks.append(CaseTask(title=task)) # Prepare the custom fields customFields = CustomFieldHelper()\ .add_string('from', fromField)\ .add_string('attachment', str(attachments))\ .build() case = Case(title=subjectField, tlp=int(config['caseTLP']), flag=False, tags=config['caseTags'], description=body, tasks=tasks, customFields=customFields) # Create the case id = None response = api.create_case(case) if response.status_code == 201: if args.verbose: print('[INFO] Created case %s' % response.json()['caseId']) else: print('[ERROR] Cannot create case: %s (%s)' % (response.status_code, response.text)) sys.exit(0) return
def export_inc_to_alert(settings, api, incident, events, rec_list): descript = ('#### Cсылка в MP SIEM: **' + settings['export_core_url'] + '/#/incident/incidents/view/' + incident["id"] + '?groupId=all_incidents&tabName=tasks**\n\n\n') if events is not None: event_counter = 0 artifacts = [] # print(json.dumps(events, indent=4, sort_keys=True)) try: for event in events: event_norm = get_event(event["id"], settings) # print(json.dumps(event, indent=4, sort_keys=True)) event_counter += 1 descript += ('### Cобытие ' + str(event_counter) + ': **' + event_norm["id"] + '**\n\n' + '|Поле | Значение |\n' + '|---------------|:-------------|\n') try: if event_norm["correlation_type"]: title_alert = event_norm["text"] except KeyError: pass observables_dict = { 'subject.name': 'other', 'object.name': 'other', 'src.ip': 'ip', 'dst.ip': 'ip', 'src.hostname': 'fqdn', 'dst.hostname': 'fqdn', 'event_src.host': 'fqdn' } for even in event_norm: if str(even) != '_meta': descript += '| **' + str(even) + '**| ' + str(event_norm[even]).replace("\n", "\\n"). \ replace("\r", "\\r").replace("|", "**OR**") + '|\n' try: artifacts.append( AlertArtifact( dataType=observables_dict[str(even)], data=event_norm[even])) except: pass # Prepare the custom fields customFields = CustomFieldHelper() \ .add_string('category', incident["category"]) \ .add_string('type', incident["type"]) \ .build() # Словарь соответствия Severity в MP SIEM и The HIVE severity_dict = {'Low': 1, 'Medium': 2, 'High': 3} alert = Alert( title=title_alert + ' (' + str(event_counter) + ' Events)', tlp=3, tags=[ incident["name"], incident["category"], incident["type"] ], description=descript, type=incident["type"], source='MP SIEM', date=(incident["created"] * 1000), sourceRef=str(incident["key"]), severity=severity_dict[incident["severity"]], artifacts=artifacts, customFields=customFields) if event_counter > 1: response = api.create_alert(alert) else: update_time_from() sys.exit(0) if response.status_code == 201: # if settings['debug'] == 1: # print(json.dumps(response.json(), indent=4, sort_keys=True)) print_log("INFO: Export incident " + incident["key"] + " to HIVE alert **" + incident["name"] + "**") write_incident_file(settings['logfile'], rec_list[-1]) else: print_log('ERROR: {}/{}'.format(response.status_code, response.text)) update_time_from() if response.status_code == 500: write_incident_file(settings['logfile'], rec_list[-1]) sys.exit(0) except: print_log("WARNING: Export incident FAILED. Waiting.") update_time_from() sys.exit(0) else: print_log("WARNING: Incident without events. Waiting.") update_time_from() sys.exit(0)
def parse_hooks(self): """ Check for new MISP Alert containing supported IOC to search automatically """ if self.webhook.isNewMispAlert(): logger.info( 'Alert {} has been tagged as MISP and is just created'.format( self.webhook.data['rootId'])) #Check alert for supported ioc types supported_iocs = False for artifact in self.webhook.data['object']['artifacts']: if artifact['dataType'] in self.qr_config[ 'supported_datatypes']: supported_iocs = True #Promote alert to case if there are support ioc types if supported_iocs: alert_id = self.webhook.data['rootId'] casetemplate = "MISP Event" logger.info('Alert {} contains IOCs that are supported'.format( alert_id)) response = self.TheHiveConnector.createCaseFromAlert( alert_id, casetemplate) self.report_action = 'createCase' """ Add timestamps to keep track of the search activity per case (we do not want to keep searching forever) """ #Perform automated Analyzer runs for supported observables in a case that has been created from a MISP alert if self.webhook.isNewMispCase(): logger.info( 'Case {} has been tagged as MISP and is just created'.format( self.webhook.data['rootId'])) #Retrieve caseid caseid = self.webhook.data['object']['id'] #Add customFields firstSearched and lastSearched #Create a Case object? Or whatever it is case = Case() #Add the case id to the object case.id = caseid #Debug output logger.info('Updating case %s' % case.id) #Define which fields need to get updated fields = ['customFields'] #Retrieve all required attributes from the alert and add them as custom fields to the case current_time = int(round(time.time() * 1000)) customFields = CustomFieldHelper()\ .add_date('firstSearched', current_time)\ .add_date('lastSearched', current_time)\ .build() #Add custom fields to the case object case.customFields = customFields #Update the case self.TheHiveConnector.updateCase(case, fields) self.report_action = 'updateCase' """ Start the analyzers automatically for MISP observables that are supported and update the case with a new timestamp """ #Automatically run Analyzers for newly created MISP cases where supported IOC's are present if self.webhook.isNewMispArtifact(): logger.info( 'Case artifact is tagged with "MISP-extern". Checking if observable is of a supported type' ) #Retrieve caseid caseid = self.webhook.data['rootId'] #Retrieve case data case_data = self.TheHiveConnector.getCase(caseid) #List all supported ioc's for the case observable = self.webhook.data['object'] #When supported, start a cortex analyzer for it if observable['dataType'] in self.qr_config['supported_datatypes']: supported_observable = observable['_id'] #Trigger a search for the supported ioc logger.info('Launching analyzers for observable: {}'.format( observable['_id'])) response = self.CortexConnector.runAnalyzer( "Cortex-intern", supported_observable, "IBMQRadar_Search_Manual_0_1") #Add customFields firstSearched and lastSearched #Create a Case object case = Case() #Add the case id to the object case.id = caseid #Debug output logger.info('Updating case %s' % case.id) #Define which fields need to get updated fields = ['customFields'] #Retrieve all required attributes from the alert and add them as custom fields to the case current_time = int(round(time.time() * 1000)) customFields = CustomFieldHelper()\ .add_date('firstSearched', case_data['customFields']['firstSearched']['date'])\ .add_date('lastSearched', current_time)\ .build() #Add custom fields to the case object case.customFields = customFields #Update the case self.TheHiveConnector.updateCase(case, fields) self.report_action = 'updateCase' """ Automatically create a task for a found IOC """ #If the Job result contains a successful search with minimum of 1 hit, create a task to investigate the results if self.webhook.isCaseArtifactJob() and self.webhook.isSuccess( ) and self.webhook.isMisp(): #Case ID caseid = self.webhook.data['rootId'] #Load Case information case_data = self.TheHiveConnector.getCase(caseid) logger.info( 'Job {} is part of a case that has been tagged as MISP case and has just finished' .format(self.webhook.data['object']['cortexJobId'])) #Check if the result count higher than 0 if int( float(self.webhook.data['object']['report']['summary'] ['taxonomies'][0]['value'])) > 0: logger.info( 'Job {} contains hits, checking if a task is already present for this observable' .format(self.webhook.data['object']['cortexJobId'])) #Retrieve case task information response = self.TheHiveConnector.getCaseTasks(caseid) case_tasks = response.json() #Load CaseTask template casetask = CaseTask() #Observable + Link observable = self.webhook.data['object']['artifactId'] observable_link = TheHive.get( 'url' ) + "/index.html#!/case/" + caseid + "/observables/" + self.webhook.data[ 'object']['artifactId'] #Task name casetask.title = "Investigate found IOC with id: {}".format( observable) #Date date_found = time.strftime("%d-%m-%Y %H:%M") case_task_found = False for case_task in case_tasks: #Check if task is present for investigating the new results if casetask.title == case_task['title']: case_task_found = True if not case_task_found: logger.info( 'No task found, creating task for observable found in job {}' .format(self.webhook.data['object']['cortexJobId'])) #Add description casetask.description = "The following ioc is hit in the environment. Investigate the results and act accordingly:\n\n" casetask.description = casetask.description + "{} is seen on {}\n".format( observable_link, date_found) #Check if case is closed if case_data['status'] == "Resolved": #Create a Case object? Or whatever it is case = Case() #Add the case id to the object case.id = caseid logger.info('Updating case %s' % case.id) #Define which fields need to get updated fields = ['status'] #Reopen the case case.status = "Open" #Update the case self.TheHiveConnector.updateCase(case, fields) #Add the case task self.TheHiveConnector.createTask(caseid, casetask) self.report_action = 'createTask'
def submitTheHive(emailFilePath): ''' Create a new case in TheHive based on the email Return 'TRUE' is successfully processed otherwise 'FALSE' ''' global log mailName, mailType = os.path.splitext(emailFilePath) if mailType == ".msg": fromField, subjectField, observables, body, attachments = readMsg(emailFilePath) elif mailType == ".eml": fromField, subjectField, observables, body, attachments = readEml(emailFilePath) # Cleanup observables (remove duplicates) new_observables = [] for o in observables: if not {'type': o['type'], 'value': o['value'] } in new_observables: # Is the observable whitelisted? if isWhitelisted(o['value']): log.debug('Skipping whitelisted observable: %s' % o['value']) else: new_observables.append({ 'type': o['type'], 'value': o['value'] }) log.debug('Found observable %s: %s' % (o['type'], o['value'])) else: log.info('Ignoring duplicate observable: %s' % o['value']) log.info("Removed duplicate observables: %d -> %d" % (len(observables), len(new_observables))) observables = new_observables api = TheHiveApi(config['thehiveURL'], config['thehiveApiKey']) # Search for interesting keywords in subjectField: log.debug("Searching for %s in '%s'" % (config['alertKeywords'], subjectField)) if re.match(config['alertKeywords'], subjectField, flags=0): # # Add observables found in the mail body # artifacts = [] if config['thehiveObservables'] and len(observables) > 0: for o in observables: artifacts.append(AlertArtifact(dataType=o['type'], data=o['value'])) # # Prepare tags - add alert keywords found to the list of tags # tags = list(config['alertTags']) match = re.findall(config['alertKeywords'], subjectField) for m in match: tags.append(m) # # Prepare the alert # sourceRef = str(uuid.uuid4())[0:6] alert = Alert(title=subjectField.replace('[ALERT]', ''), tlp = int(config['alertTLP']), tags = tags, description = body, type = 'external', source = fromField, sourceRef = sourceRef, artifacts = artifacts) # Create the Alert id = None response = api.create_alert(alert) if response.status_code == 201: log.info('Created alert %s' % response.json()['sourceRef']) else: log.error('Cannot create alert: %s (%s)' % (response.status_code, response.text)) return False else: # Prepare the sample case tasks = [] for task in config['caseTasks']: tasks.append(CaseTask(title=task)) # Prepare the custom fields customFields = CustomFieldHelper()\ .add_string('from', fromField)\ .add_string('attachment', str(attachments))\ .build() # If a case template is specified, use it instead of the tasks if len(config['caseTemplate']) > 0: case = Case(title=subjectField, tlp = int(config['caseTLP']), flag = False, tags = config['caseTags'], description = body, template = config['caseTemplate'], customFields = customFields) else:
# Define variables HIVE_SERVER_IP = configuration.SYSTEM_SETTINGS['HIVE_SERVER_IP'] HIVE_SERVER_PORT = configuration.SYSTEM_SETTINGS['HIVE_SERVER_PORT'] HIVE_API_KEY = configuration.SYSTEM_SETTINGS['HIVE_API_KEY'] api = TheHiveApi("http://%s:%s" % (HIVE_SERVER_IP, HIVE_SERVER_PORT), HIVE_API_KEY) # Prepare observables artifacts = [ AlertArtifact(dataType='ip', data='8.8.8.8'), AlertArtifact(dataType='domain', data='google.com'), AlertArtifact(dataType='domain', data='test'), AlertArtifact(dataType='domain', data='sample.txt', sighted=True, ioc=True) ] # Prepare custom fields customFields = CustomFieldHelper()\ .build() # Prepare the sample Alert sourceRef = str(uuid.uuid4())[0:6] alert = Alert(title='Tier3 Test', tlp=3, severity=3, tags=['TheHive4Py', 'sample'], description='N/A', type='external', source='instance1', sourceRef=sourceRef, artifacts=artifacts, customFields=customFields) # Create the alert
def doWork(self): #Build a loop that keeps the thread alive until queue is empty while not q.empty(): #Build up the threads thread_count = threading.active_count() #Make sure that the thread count is lower than configured limit and is lower than the queue size if thread_count < concurrent and thread_count < q.qsize(): new_thread_count = thread_count + 1 logger.info( 'Current queue size(%i) allows more threads. Creating additional thread: %i' % (q.qsize(), new_thread_count)) t = Thread(target=self.doWork) t.daemon = True t.start() logger.debug('Created thread: %i' % new_thread_count) #Retrieve a queued item queued_item = q.get() #Handle a queued item based on its provided action if queued_item['action'] == "search_observables": logger.info('Working on %s from queue, caseid: %s' % (queued_item['action'], queued_item['data']['id'])) case_data = queued_item['data'] logger.debug("event: %s" % case_data) #Store CaseID caseid = case_data['id'] #If the case is within scope of the search range. Perform the search #if (case_data['customFields']['lastSearched']['date'] - case_data['customFields']['firstSearched']['date']) < 5184000000: #logger.info('Observables in case {} have not yet been searched for longer than two months. Starting analyzers'.format(case_data['id'])) self.response = self.thapi.get_case_observables(caseid) #Perform a search for ioc's per case in the RS search results (Trigger Cortex Analyzer) for observable in self.response.json(): searched_for = False logger.debug("observable: %s" % observable) logger.debug("current_time %s, observable_time %s" % (self.current_time, observable['startDate'])) #Check if observables are not older than 2 months or 6 months for TLP:RED if (((self.current_time - observable['startDate']) < 5184000000) or (observable['tlp'] == 3 and ((self.current_time - observable['startDate']) < 15552000))): self.searchtype = observable['dataType'] if self.searchtype in self.qr_enabled_datatypes: self.supported_observable = observable['_id'] #Trigger a search for the supported ioc logger.info( 'Launching analyzers for observable: {}'. format(self.supported_observable)) self.response = self.thapi.run_analyzer( "Cortex-intern", self.supported_observable, "IBMQRadar_Search_Automated_0_1") self.check_response(self.response) searched_for = True if searched_for: #Add customFields firstSearched and lastSearched #Create a Case object? Or whatever it is self.case = Case() #Add the case id to the object self.case.id = caseid #Debug output logger.info('Updating case %s' % self.case.id) #Define which fields need to get updated fields = ['customFields'] #Retrieve all required attributes from the alert and add them as custom fields to the case self.customFields = CustomFieldHelper()\ .add_date('firstSearched', case_data['customFields']['firstSearched']['date'])\ .add_date('lastSearched', self.current_time)\ .build() #Add custom fields to the case object self.case.customFields = self.customFields #Update the case self.response = self.thapi.update_case(self.case, fields) self.check_response(self.response) logger.info("Queue is empty, nothing left to do")