class Automators(Main): def __init__(self, cfg, use_case_config): self.logger = logging.getLogger(__name__) self.logger.info('Initiating QRadar Automators') self.cfg = cfg self.use_case_config = use_case_config self.TheHiveConnector = TheHiveConnector(cfg) self.TheHiveAutomators = TheHiveAutomators(cfg, use_case_config) self.QRadarConnector = QRadarConnector(cfg) def checkSiem(self, action_config, webhook): #Only continue if the right webhook is triggered if webhook.isImportedAlert() or webhook.isNewAlert() or webhook.isQRadarAlertUpdateFollowTrue(): pass else: return False #Define variables and actions based on certain webhook types #Alerts if webhook.isNewAlert() or webhook.isQRadarAlertUpdateFollowTrue(): self.alert_id = webhook.data['object']['id'] self.alert_description = webhook.data['object']['description'] self.supported_query_type = 'enrichment_queries' #Cases elif webhook.isImportedAlert(): self.case_id = webhook.data['object']['case'] self.supported_query_type = 'search_queries' self.query_variables = {} self.query_variables['input'] = {} self.enriched = False #Prepare search queries for searches for query_name, query_config in action_config[self.supported_query_type].items(): try: self.logger.info('Found the following query: %s' % (query_name)) self.query_variables[query_name] = {} #Render query try: #Prepare the template self.template = Template(query_config['query']) #Find variables in the template self.template_env = Environment() self.template_parsed = self.template_env.parse(query_config['query']) #Grab all the variales from the template and try to find them in the description self.template_vars = meta.find_undeclared_variables(self.template_parsed) self.logger.debug("Found the following variables in query: {}".format(self.template_vars)) for template_var in self.template_vars: #Skip dynamically generated Stop_time variable if template_var == "Stop_Time": continue self.logger.debug("Looking up variable required for template: {}".format(template_var)) #Replace the underscore from the variable name to a white space as this is used in the description table self.template_var_with_ws = template_var.replace("_", " ") self.query_variables['input'][template_var] = self.TheHiveAutomators.fetchValueFromDescription(webhook,self.template_var_with_ws) #Parse times required for the query (with or without offset) if template_var == "Start_Time": self.logger.debug("Found Start Time: %s" % self.query_variables['input']['Start_Time']) if 'start_time_offset' in query_config: self.query_variables['input']['Start_Time'] = self.parseTimeOffset(self.query_variables['input']['Start_Time'], self.cfg.get('Automation', 'event_start_time_format'), query_config['start_time_offset'], self.cfg.get('QRadar', 'time_format')) else: self.query_variables['input']['Start_Time'] = self.query_variables['input']['Start_Time'] if 'stop_time_offset' in query_config: self.query_variables['input']['Stop_Time'] = self.parseTimeOffset(self.query_variables['input']['Start_Time'], self.cfg.get('Automation', 'event_start_time_format'), query_config['stop_time_offset'], self.cfg.get('QRadar', 'time_format')) else: self.query_variables['input']['Stop_Time'] = datetime.now().strftime(self.cfg.get('Automation', 'event_start_time_format')) if not self.query_variables['input']['Start_Time']: self.logger.warning("Could not find Start Time value ") raise GetOutOfLoop self.query_variables[query_name]['query'] = self.template.render(self.query_variables['input']) self.logger.debug("Rendered the following query: %s" % self.query_variables[query_name]['query']) except Exception as e: self.logger.warning("Could not render query due to missing variables", exc_info=True) raise GetOutOfLoop #Perform search queries try: self.query_variables[query_name]['result'] = self.QRadarConnector.aqlSearch(self.query_variables[query_name]['query']) except Exception as e: self.logger.warning("Could not perform query", exc_info=True) raise GetOutOfLoop #Check results self.logger.debug('The search result returned the following information: \n %s' % self.query_variables[query_name]['result']) if self.supported_query_type == "search_queries": #Task name self.uc_task_title = query_config['task_title'] self.uc_task_description = "The following information is found. Investigate the results and act accordingly:\n\n\n\n" #create a table header self.table_header = "|" self.rows = "|" if len(self.query_variables[query_name]['result']['events']) != 0: for key in self.query_variables[query_name]['result']['events'][0].keys(): self.table_header = self.table_header + " %s |" % key self.rows = self.rows + "---|" self.table_header = self.table_header + "\n" + self.rows + "\n" self.uc_task_description = self.uc_task_description + self.table_header #Create the data table for the results for event in self.query_variables[query_name]['result']['events']: self.table_data_row = "|" for field_key, field_value in event.items(): # Escape pipe signs if field_value: field_value = field_value.replace("|", "|") # Use to create some additional spacing self.table_data_row = self.table_data_row + " %s |" % field_value self.table_data_row = self.table_data_row + "\n" self.uc_task_description = self.uc_task_description + self.table_data_row else: self.uc_task_description = self.uc_task_description + "No results \n" #Add the case task self.uc_task = self.TheHiveAutomators.craftUcTask(self.uc_task_title, self.uc_task_description) self.TheHiveConnector.createTask(self.case_id, self.uc_task) if self.supported_query_type == "enrichment_queries": #Add results to description try: if self.TheHiveAutomators.fetchValueFromDescription(webhook,query_name) != self.query_variables[query_name]['result']['events'][0]['enrichment_result']: self.regex_end_of_table = ' \|\\n\\n\\n' self.end_of_table = ' |\n\n\n' self.replacement_description = '|\n | **%s** | %s %s' % (query_name, self.query_variables[query_name]['result']['events'][0]['enrichment_result'], self.end_of_table) self.th_alert_description = self.TheHiveConnector.getAlert(self.alert_id)['description'] self.alert_description = re.sub(self.regex_end_of_table, self.replacement_description, self.th_alert_description) self.enriched = True #Update Alert with the new description field self.updated_alert = Alert self.updated_alert.description = self.alert_description self.TheHiveConnector.updateAlert(self.alert_id, self.updated_alert, ["description"]) except Exception as e: self.logger.warning("Could not add results from the query to the description. Error: {}".format(e)) raise GetOutOfLoop except GetOutOfLoop: pass return True
class Integration(Main): def __init__(self): super().__init__() self.lexsi = LexsiConnector(self.cfg) self.theHiveConnector = TheHiveConnector(self.cfg) def validateRequest(self, request): if request.is_json: self.content = request.get_json() if 'type' in self.content and self.content['type'] == "Active": self.workflowReport = self.allIncidents2Alert( self.content['type']) if self.workflowReport['success']: return json.dumps(self.workflowReport), 200 else: return json.dumps(self.workflowReport), 500 else: self.logger.error('Missing type or type is not supported') return json.dumps({ 'sucess': False, 'message': "Missing type or type is not supported" }), 500 else: self.logger.error('Not json request') return json.dumps({ 'sucess': False, 'message': "Request didn't contain valid JSON" }), 400 def allIncidents2Alert(self, status): """ Get all opened incidents created within lexsi and create alerts for them in TheHive """ self.logger.info('%s.allincident2Alert starts', __name__) self.incidentsList = self.lexsi.getOpenItems()['result'] self.report = dict() self.report['success'] = True self.report['incidents'] = list() try: # each incidents in the list is represented as a dict # we enrich this dict with additional details for incident in self.incidentsList: # Prepare new alert self.incident_report = dict() self.logger.debug("incident: %s" % incident) self.theHiveAlert = self.IncidentToHiveAlert(incident) # searching if the incident has already been converted to alert self.query = dict() self.query['sourceRef'] = str(incident['incident']) self.logger.info('Looking for incident %s in TheHive alerts', str(incident['incident'])) self.results = self.theHiveConnector.findAlert(self.query) if len(self.results) == 0: self.logger.info( 'incident %s not found in TheHive alerts, creating it', str(incident['incident'])) try: self.theHiveEsAlertId = self.theHiveConnector.createAlert( self.theHiveAlert)['id'] self.theHiveConnector.promoteCaseToAlert( self.theHiveEsAlertId) self.incident_report[ 'raised_alert_id'] = self.theHiveEsAlertId self.incident_report['lexsi_incident_id'] = incident[ 'incident'] self.incident_report['success'] = True except Exception as e: self.logger.error(self.incident_report) self.logger.error('%s.allincident2Alert failed', __name__, exc_info=True) self.incident_report['success'] = False if isinstance(e, ValueError): errorMessage = json.loads(str(e))['message'] self.incident_report['message'] = errorMessage else: self.incident_report['message'] = str( e) + ": Couldn't raise alert in TheHive" self.incident_report['incident_id'] = incident[ 'incident'] # Set overall success if any fails self.report['success'] = False self.report['incidents'].append(self.incident_report) else: self.logger.info( 'incident %s already imported as alert, checking for updates', str(incident['incident'])) self.alert_found = self.results[0] # Check if alert is already created, but needs updating if self.check_if_updated(self.alert_found, vars(self.theHiveAlert)): self.logger.info( "Found changes for %s, updating alert" % self.alert_found['id']) # update alert self.theHiveConnector.updateAlert( self.alert_found['id'], self.theHiveAlert, fields=["tags", "artifacts"]) # Mark the alert as read self.theHiveConnector.markAlertAsRead( self.alert_found['id']) self.incident_report[ 'updated_alert_id'] = self.alert_found['id'] self.incident_report['lexsi_incident_id'] = incident[ 'incident'] self.incident_report['success'] = True else: self.logger.info("No changes found for %s" % self.alert_found['id']) continue self.thehiveAlerts = self.lexsi_opened_alerts_thehive() self.set_alert_status_ignored(self.incidentsList) except Exception as e: self.logger.error( 'Failed to create alert from Lexsi incident (retrieving incidents failed)', exc_info=True) self.report['success'] = False self.report[ 'message'] = "%s: Failed to create alert from incident" % str( e) return self.report def IncidentToHiveAlert(self, incident): # # Creating the alert # # Setup Tags self.tags = ['Lexsi', 'incident', 'Synapse'] # Skip for now self.artifacts = [] # Retrieve the configured case_template self.CaseTemplate = self.cfg.get('Lexsi', 'case_template') # Build TheHive alert self.alert = self.theHiveConnector.craftAlert( "{}: {}".format(incident['incident'], incident['title']), self.craftAlertDescription(incident), self.getHiveSeverity(incident), self.timestamp_to_epoch(incident['detected'], "%Y-%m-%d %H:%M:%S"), self.tags, 2, 'New', 'internal', 'Lexsi', str(incident['incident']), self.artifacts, self.CaseTemplate) return self.alert def craftAlertDescription(self, incident): """ From the incident metadata, crafts a nice description in markdown for TheHive """ self.logger.debug('craftAlertDescription starts') # Start empty self.description = "" # Add incident details table self.description += ( '#### Summary\n\n' + '| | |\n' + '| ----------------------- | ------------- |\n' + '| **URL** | ' + "{}{}{}".format("```", str(incident['url']), "```") + ' |\n' + '| **Type** | ' + str(incident['type']) + ' |\n' + '| **Severity** | ' + str(incident['severity']) + ' |\n' + '| **Category** | ' + str(incident['category']) + ' |\n' + '| **Updated** | ' + str(incident['updated']) + ' |\n' + '| **Detected** | ' + str(incident['detected']) + ' |\n' + '| **Source** | ' + str(incident['source']) + ' |\n' + '| **Analyst Name(Lexsi)** | ' + str(incident['analystName']) + ' |\n' + '| **Link to Orange Portal** | ' + str("https://cert.orangecyberdefense.com/#cyb/alerts/{}".format( incident['incident'])) + ' |\n' + '\n\n\n\n') return self.description def timestamp_to_epoch(self, date_time, pattern): return int(time.mktime(time.strptime(date_time, pattern))) * 1000 def getHiveSeverity(self, incident): # severity in TheHive is either low, medium, high or critical # while severity in Lexsi is from 0 to 5 if int(incident['severity']) in {0, 5}: return 1 # elif int(incident['severity']) in {2,3}: # return 2 # elif int(incident['severity']) in {4,5}: # return 3 def lexsi_opened_alerts_thehive(self): self.thehiveAlerts = [] self.open_lexsi_cases = {} self.query = In('tags', ['Lexsi']) self.logger.info( 'Looking for incident in TheHive alerts with tag Lexsi') # self.logger.info(self.query) self.results = self.theHiveConnector.findAlert(self.query) for alert_found in self.results: # Check if a case is linked if 'case' in alert_found: try: self.case_found = self.theHiveConnector.getCase( alert_found['case']) # Check if the status is open. Only then append it to the list if self.case_found['status'] == "Open": self.open_lexsi_cases[ alert_found['sourceRef']] = self.case_found self.thehiveAlerts.append(alert_found['sourceRef']) except Exception as e: self.logger.error("Could not find case: {}".format(e), exc_info=True) continue self.logger.debug("Lexsi Alerts opened in theHive: {}".format( self.thehiveAlerts)) return self.thehiveAlerts def compare_lists(self, list1, list2): return list(set(list1) - set(list2)) def set_alert_status_ignored(self, incidentsList): self.lexsi_reporting = [] # self.incidentsList = self.lexsi.getOpenItems()['result'] for incident in incidentsList: self.lexsi_reporting.append(incident['incident']) self.logger.debug("the list of opened Lexsi Incidents: {}".format( self.lexsi_reporting)) self.uncommon_elements = self.compare_lists(self.thehiveAlerts, self.lexsi_reporting) # self.uncommon_elements=['476121'] self.logger.debug( "Open cases present in TheHive but not in list of opened Lexsi Incidents: {}" .format((self.uncommon_elements))) for element in self.uncommon_elements: self.logger.info( "Preparing to close the case for {}".format(element)) self.query = dict() self.query['sourceRef'] = str(element) self.logger.debug('Looking for incident %s in TheHive alerts', str(element)) try: if element in self.open_lexsi_cases: # Resolve the case self.case_id = self.open_lexsi_cases[element]['id'] self.logger.debug("Case id for element {}: {}".format( element, self.case_id)) self.logger.debug("Preparing to resolve the case") self.theHiveConnector.closeCase(self.case_id) self.logger.debug("Closed case with id {} for {}".format( self.case_id, element)) except Exception as e: self.logger.error("Could not close case: {}".format(e), exc_info=True) continue
class Integration(Main): def __init__(self): super().__init__() self.azureSentinelConnector = AzureSentinelConnector(self.cfg) self.theHiveConnector = TheHiveConnector(self.cfg) def craftAlertDescription(self, incident): """ From the incident metadata, crafts a nice description in markdown for TheHive """ self.logger.debug('craftAlertDescription starts') # Start empty self.description = "" # Add url to incident self.url = ('[%s](%s)' % (str(incident['properties']['incidentNumber']), str(incident['properties']['incidentUrl']))) self.description += '#### Incident: \n - ' + self.url + '\n\n' # Format associated rules self.rule_names_formatted = "#### Rules triggered: \n" self.rules = incident['properties']['relatedAnalyticRuleIds'] if len(self.rules) > 0: for rule in self.rules: self.rule_info = self.azureSentinelConnector.getRule(rule) self.logger.debug( 'Received the following rule information: {}'.format( self.rule_info)) self.rule_name = self.rule_info['properties']['displayName'] rule_url = "https://management.azure.com{}".format(rule) self.rule_names_formatted += "- %s \n" % (self.rule_name) # Add rules overview to description self.description += self.rule_names_formatted + '\n\n' # Add mitre Tactic information # https://raw.githubusercontent.com/mitre/cti/master/enterprise-attack/enterprise-attack.json # mitre_ta_links_formatted = "#### MITRE Tactics: \n" # if 'mitre_tactics' in offense and offense['mitre_tactics']: # for tactic in offense['mitre_tactics']: # mitre_ta_links_formatted += "- [%s](%s/%s) \n" % (tactic, 'https://attack.mitre.org/tactics/', tactic) # #Add associated documentation # self.description += mitre_ta_links_formatted + '\n\n' # #Add mitre Technique information # mitre_t_links_formatted = "#### MITRE Techniques: \n" # if 'mitre_techniques' in offense and offense['mitre_techniques']: # for technique in offense['mitre_techniques']: # mitre_t_links_formatted += "- [%s](%s/%s) \n" % (technique, 'https://attack.mitre.org/techniques/', technique) # Add a custom description when the incident does not contain any if 'description' not in incident['properties']: incident['properties']['description'] = "N/A" # Add incident details table self.description += ( '#### Summary\n\n' + '| | |\n' + '| ----------------------- | ------------- |\n' + '| **Start Time** | ' + str( self.azureSentinelConnector.formatDate( "description", incident['properties']['createdTimeUtc'])) + ' |\n' + '| **incident ID** | ' + str(incident['properties']['incidentNumber']) + ' |\n' + '| **Description** | ' + str(incident['properties']['description'].replace('\n', '')) + ' |\n' + '| **incident Type** | ' + str(incident['type']) + ' |\n' + '| **incident Source** | ' + str(incident['properties']['additionalData']['alertProductNames']) + ' |\n' + '| **incident Status** | ' + str(incident['properties']['status']) + ' |\n' + '\n\n\n\n') return self.description def sentinelIncidentToHiveAlert(self, incident): def getHiveSeverity(incident): # severity in TheHive is either low, medium or high # while severity in Sentinel is from Low to High if incident['properties']['severity'] == "Low": return 1 elif incident['properties']['severity'] == "Medium": return 2 elif incident['properties']['severity'] == "High": return 3 return 1 # # Creating the alert # # Setup Tags self.tags = ['AzureSentinel', 'incident', 'Synapse'] # Skip for now self.artifacts = [] # Retrieve the configured case_template self.sentinelCaseTemplate = self.cfg.get('AzureSentinel', 'case_template') # Build TheHive alert self.alert = self.theHiveConnector.craftAlert( "{}, {}".format(incident['properties']['incidentNumber'], incident['properties']['title']), self.craftAlertDescription(incident), getHiveSeverity(incident), self.azureSentinelConnector.formatDate( "alert_timestamp", incident['properties']['createdTimeUtc']), self.tags, 2, 'New', 'internal', 'Azure_Sentinel_incidents', str(incident['name']), self.artifacts, self.sentinelCaseTemplate) return self.alert def validateRequest(self, request): if request.is_json: self.content = request.get_json() if 'type' in self.content and self.content['type'] == "Active": self.workflowReport = self.allIncidents2Alert( self.content['type']) if self.workflowReport['success']: return json.dumps(self.workflowReport), 200 else: return json.dumps(self.workflowReport), 500 else: self.logger.error('Missing type or type is not supported') return json.dumps({ 'sucess': False, 'message': "Missing type or type is not supported" }), 500 else: self.logger.error('Not json request') return json.dumps({ 'sucess': False, 'message': "Request didn't contain valid JSON" }), 400 def allIncidents2Alert(self, status): """ Get all opened incidents created within Azure Sentinel and create alerts for them in TheHive """ self.logger.info('%s.allincident2Alert starts', __name__) self.report = dict() self.report['success'] = True self.report['incidents'] = list() try: self.incidentsList = self.azureSentinelConnector.getIncidents() # each incidents in the list is represented as a dict # we enrich this dict with additional details for incident in self.incidentsList: # Prepare new alert self.incident_report = dict() self.logger.debug("incident: %s" % incident) # self.logger.info("Enriching incident...") # enrichedincident = enrichIncident(incident) # self.logger.debug("Enriched incident: %s" % enrichedincident) self.theHiveAlert = self.sentinelIncidentToHiveAlert(incident) # searching if the incident has already been converted to alert self.query = dict() self.query['sourceRef'] = str(incident['name']) self.logger.info('Looking for incident %s in TheHive alerts', str(incident['name'])) self.results = self.theHiveConnector.findAlert(self.query) if len(self.results) == 0: self.logger.info( 'incident %s not found in TheHive alerts, creating it', str(incident['name'])) try: self.theHiveEsAlertId = self.theHiveConnector.createAlert( self.theHiveAlert)['id'] self.incident_report[ 'raised_alert_id'] = self.theHiveEsAlertId self.incident_report[ 'sentinel_incident_id'] = incident['name'] self.incident_report['success'] = True except Exception as e: self.logger.error('%s.allincident2Alert failed', __name__, exc_info=True) self.incident_report['success'] = False if isinstance(e, ValueError): errorMessage = json.loads(str(e))['message'] self.incident_report['message'] = errorMessage else: self.incident_report['message'] = str( e) + ": Couldn't raise alert in TheHive" self.incident_report['incident_id'] = incident['name'] # Set overall success if any fails self.report['success'] = False self.report['incidents'].append(self.incident_report) else: self.logger.info( 'incident %s already imported as alert, checking for updates', str(incident['name'])) self.alert_found = self.results[0] # Check if alert is already created, but needs updating if self.check_if_updated(self.alert_found, vars(self.theHiveAlert)): self.logger.info( "Found changes for %s, updating alert" % self.alert_found['id']) # update alert self.theHiveConnector.updateAlert( self.alert_found['id'], self.theHiveAlert, fields=["tags", "artifacts"]) self.incident_report[ 'updated_alert_id'] = self.alert_found['id'] self.incident_report[ 'sentinel_incident_id'] = incident['name'] self.incident_report['success'] = True else: self.logger.info("No changes found for %s" % self.alert_found['id']) continue except Exception as e: self.logger.error( 'Failed to create alert from Azure Sentinel incident (retrieving incidents failed)', exc_info=True) self.report['success'] = False self.report[ 'message'] = "%s: Failed to create alert from incident" % str( e) return self.report
class Integration(Main): def __init__(self): super().__init__() self.RDConnector = RDConnector(self.cfg) self.TheHiveConnector = TheHiveConnector(self.cfg) def validateRequest(self, request): workflowReport = self.connectRD() if workflowReport['success']: return json.dumps(workflowReport), 200 else: return json.dumps(workflowReport), 500 def connectRD(self): self.logger.info('%s.connectResponsibleDisclosure starts', __name__) report = dict() report['success'] = bool() # Setup Tags self.tags = ['Responsible disclosure', 'Synapse'] tracker_file = "./modules/ResponsibleDisclosure/email_tracker" link_to_load = "" if os.path.exists(tracker_file): self.logger.debug("Reading from the tracker file...") with open(tracker_file, "r") as tracker: link_to_load = tracker.read() if not link_to_load: link_to_load = self.cfg.get('ResponsibleDisclosure', 'list_endpoint') emails, new_link = self.RDConnector.scan(link_to_load) try: for email in emails: try: if ('@removed' in email) or [email["from"]["emailAddress"]["address"]] in self.cfg.get('ResponsibleDisclosure', 'excluded_senders'): continue self.logger.debug("Found unread E-mail with id: {}".format(email['id'])) # Get the conversation id from the email CID = email["conversationId"] # Conversation id hash will be used as a unique identifier for the alert CIDHash = hashlib.md5(CID.encode()).hexdigest() email_date = datetime.strptime(email["receivedDateTime"], "%Y-%m-%dT%H:%M:%SZ") epoch_email_date = email_date.timestamp() * 1000 alertTitle = "Responsible Disclosure - {}".format(email["subject"]) alertDescription = self.createDescription(email) # Moving the email from Inbox to the new folder defined by variable to_move_folder in synapse.conf # Disabled temporarily # self.RDConnector.moveToFolder(self.cfg.get('ResponsibleDisclosure', 'email_address'), email['id'], self.cfg.get('ResponsibleDisclosure', 'to_move_folder')) # Get all the attachments and upload to the hive observables attachment_data = self.RDConnector.listAttachment(self.cfg.get('ResponsibleDisclosure', 'email_address'), email['id']) all_artifacts = [] all_attachments = [] if attachment_data: for att in attachment_data: file_name = self.RDConnector.downloadAttachments(att['name'], att['attachment_id'], att['isInline'], att['contentType']) all_attachments.append(file_name) self.af = AlertArtifact(dataType='file', data=file_name, tlp=2, tags=['Responsible disclosure', 'Synapse'], ioc=True) all_artifacts.append(self.af) # Create the alert in thehive alert = self.TheHiveConnector.craftAlert( alertTitle, alertDescription, 1, epoch_email_date, self.tags, 2, "New", "internal", "ResponsibleDisclosure", CIDHash, all_artifacts, self.cfg.get('ResponsibleDisclosure', 'case_template')) # Check if the alert was created successfully query = dict() query['sourceRef'] = str(CIDHash) # Look up if any existing alert in theHive alert_results = self.TheHiveConnector.findAlert(query) # If no alerts are found for corresponding CIDHASH, create a new alert if len(alert_results) == 0: createdAlert = self.TheHiveConnector.createAlert(alert) # automatish antwoord to the original email sender from the responsible disclosure emailaddress autoreply_subject_name = "RE: {}".format(email["subject"]) self.RDConnector.sendAutoReply("*****@*****.**", email["from"]["emailAddress"]["address"], self.cfg.get('ResponsibleDisclosure', 'email_body_filepath'), autoreply_subject_name) # If alert is found update the alert or it may have been migrated to case so update the case if len(alert_results) > 0: alert_found = alert_results[0] # Check if alert is promoted to a case if 'case' in alert_found: case_found = self.TheHiveConnector.getCase(alert_found['case']) # Create a case model self.updated_case = Case # Update the case with new description # What if the email body is empty for new email, then use the old description self.updated_case.description = case_found['description'] + "\n\n" + alertDescription self.updated_case.id = alert_found['case'] self.TheHiveConnector.updateCase(self.updated_case, ["description"]) self.logger.info("updated the description of the case with id: {}".format(alert_found['case'])) # Check if there new observables available if all_attachments: for att in all_attachments: try: self.TheHiveConnector.addFileObservable(alert_found['case'], att, "email attachment") except Exception as e: self.logger.error(f"Encountered an error while creating a new file based observable: {e}", exc_info=True) continue # Else it means there is no corresponding case so update the alert else: # create an alert model self.updated_alert = Alert # Update the alert with new description # What if the email body is empty for new email, then use the old description self.updated_alert.description = alert_found['description'] + "\n\n" + alertDescription self.TheHiveConnector.updateAlert(alert_found['id'], self.updated_alert, ["description"]) self.logger.info("updated the description of the alert with id: {}".format(alert_found['id'])) except Exception as e: self.logger.error(e, exc_info=True) continue if all_attachments: for att in all_attachments: os.remove(att) # Write the delta link to the tracker with open(tracker_file, "w+") as tracker: tracker.write(new_link) report['success'] = True return report except Exception as e: self.logger.error(e) self.logger.error('Connection failure', exc_info=True) report['success'] = False return report def createDescription(self, email): email_body = email['body']['content'] subject = email["subject"] # Get the conversation id from the email CID = email["conversationId"] # Conversation id hash will be used as a unique identifier for the alert CIDHash = hashlib.md5(CID.encode()).hexdigest() # Parse all the URLs and add them to a field in the description table urls_list = re.findall(r'\<(https?://[\S]+?)\>', email_body) # " " is ascii for next line urls_str = ' '.join(str(x) for x in urls_list) from_e = email["from"]["emailAddress"]["address"] to_e = "N/A" if email["toRecipients"]: to_e = email["toRecipients"][0]["emailAddress"]["address"] OriginatingIP = "N/A" for header in email['internetMessageHeaders']: if header['name'] == 'X-Originating-IP': # Formatting the ip value, bydefault it comesup like [x.x.x.x] OriginatingIP = (header['value'][1:-1]) # putting together the markdown table temp_fullbody = [] temp_fullbody.append("| | |") temp_fullbody.append("|:-----|:-----|") temp_fullbody.append("| " + "**" + "Subject" + "**" + " | " + subject + " |") temp_fullbody.append("| " + "**" + "Sender" + "**" + " | " + from_e + " |") temp_fullbody.append("| " + "**" + "Recipient" + "**" + " | " + to_e + " |") temp_fullbody.append("| " + "**" + "Originating IP" + "**" + " | " + OriginatingIP + " |") temp_fullbody.append("| " + "**" + "Received at" + "**" + " | " + email["receivedDateTime"] + " |") temp_fullbody.append("| " + "**" + "URL(s) in email" + "**" + " | " + urls_str + " |") temp_fullbody.append("| " + "**" + "Msg ID" + "**" + " | " + email['id'] + " |") temp_fullbody.append("**" + "Email body" + "**") temp_fullbody.append("```") temp_fullbody.append(email_body) temp_fullbody.append("```") alertDescription = '\r\n'.join(str(x) for x in temp_fullbody) return alertDescription
class Integration(Main): def __init__(self): super().__init__() self.qradarConnector = QRadarConnector(self.cfg) self.theHiveConnector = TheHiveConnector(self.cfg) def enrichOffense(self, offense): self.enriched = copy.deepcopy(offense) self.artifacts = [] self.enriched['offense_type_str'] = \ self.qradarConnector.getOffenseTypeStr(offense['offense_type']) # Add the offense source explicitly if self.enriched['offense_type_str'] == 'Username': self.artifacts.append({ 'data': offense['offense_source'], 'dataType': 'user-account', 'message': 'Offense Source', 'tags': ['QRadar'] }) # Add the local and remote sources # scrIps contains offense source IPs self.srcIps = list() # dstIps contains offense destination IPs self.dstIps = list() # srcDstIps contains IPs which are both source and destination of offense self.srcDstIps = list() for ip in self.qradarConnector.getSourceIPs(self.enriched): self.srcIps.append(ip) for ip in self.qradarConnector.getLocalDestinationIPs(self.enriched): self.dstIps.append(ip) # making copies is needed since we want to # access and delete data from the list at the same time self.s = copy.deepcopy(self.srcIps) self.d = copy.deepcopy(self.dstIps) for srcIp in self.s: for dstIp in self.d: if srcIp == dstIp: self.srcDstIps.append(srcIp) self.srcIps.remove(srcIp) self.dstIps.remove(dstIp) for ip in self.srcIps: self.artifacts.append({ 'data': ip, 'dataType': 'ip', 'message': 'Source IP', 'tags': ['QRadar', 'src'] }) for ip in self.dstIps: self.artifacts.append({ 'data': ip, 'dataType': 'ip', 'message': 'Local destination IP', 'tags': ['QRadar', 'dst'] }) for ip in self.srcDstIps: self.artifacts.append({ 'data': ip, 'dataType': 'ip', 'message': 'Source and local destination IP', 'tags': ['QRadar', 'src', 'dst'] }) # Parse offense types to add the offense source as an observable when a valid type is used for offense_type, extraction_config in self.cfg.get( 'QRadar', 'observables_in_offense_type', fallback={}).items(): if self.enriched['offense_type_str'] == offense_type: if isinstance(extraction_config, str): self.observable_type = extraction_config self.artifacts.append({ 'data': self.enriched['offense_source'], 'dataType': self.observable_type, 'message': 'QRadar Offense source', 'tags': ['QRadar'] }) elif isinstance(extraction_config, list): for extraction in extraction_config: self.regex = re.compile(extraction['regex']) self.matches = self.regex.findall( str(self.enriched['offense_source'])) if len(self.matches) > 0: # if isinstance(found_observable, tuple): << Fix later loop through matches as well for match_group, self.observable_type in extraction[ 'match_groups'].items(): try: self.artifacts.append({ 'data': self.matches[0][match_group], 'dataType': self.observable_type, 'message': 'QRadar Offense Type based observable', 'tags': ['QRadar', 'offense_type'] }) except Exception as e: self.logger.warning( "Could not find match group {} in {}". format( match_group, self.enriched['offense_type_str'])) else: self.logger.error( "Configuration for observables_in_offense_type is wrongly formatted. Please fix this to enable this functionality" ) # Remove observables that are to be excluded based on the configuration self.artifacts = self.checkObservableExclusionList(self.artifacts) # Match observables against the TLP list self.artifacts = self.checkObservableTLP(self.artifacts) # Add all the observables self.enriched['artifacts'] = self.artifacts # Add rule names to offense self.enriched['rules'] = self.qradarConnector.getRuleNames(offense) # waiting 1s to make sure the logs are searchable sleep(1) # adding the first 3 raw logs self.enriched['logs'] = self.qradarConnector.getOffenseLogs( self.enriched) return self.enriched def qradarOffenseToHiveAlert(self, offense): def getHiveSeverity(offense): # severity in TheHive is either low, medium or high # while severity in QRadar is from 1 to 10 # low will be [1;4] => 1 # medium will be [5;6] => 2 # high will be [7;10] => 3 if offense['severity'] < 5: return 1 elif offense['severity'] < 7: return 2 elif offense['severity'] < 11: return 3 return 1 # # Creating the alert # # Setup Tags self.tags = ['QRadar', 'Offense', 'Synapse'] # Add the offense type as a tag if 'offense_type_str' in offense: self.tags.append("qr_offense_type: {}".format( offense['offense_type_str'])) # Check if the automation ids need to be extracted if self.cfg.getboolean('QRadar', 'extract_automation_identifiers'): # Run the extraction function and add it to the offense data # Extract automation ids self.tags_extracted = self.tagExtractor( offense, self.cfg.get('QRadar', 'automation_fields'), self.cfg.get('QRadar', 'tag_regexes')) # Extract any possible name for a document on a knowledge base offense['use_case_names'] = self.tagExtractor( offense, self.cfg.get('QRadar', 'automation_fields'), self.cfg.get('QRadar', 'uc_kb_name_regexes')) if len(self.tags_extracted) > 0: self.tags.extend(self.tags_extracted) else: self.logger.info('No match found for offense %s', offense['id']) # Check if the mitre ids need to be extracted if self.cfg.getboolean('QRadar', 'extract_mitre_ids'): # Extract mitre tactics offense['mitre_tactics'] = self.tagExtractor( offense, ["rules"], ['[tT][aA]\d{4}']) if 'mitre_tactics' in offense: self.tags.extend(offense['mitre_tactics']) # Extract mitre techniques offense['mitre_techniques'] = self.tagExtractor( offense, ["rules"], ['[tT]\d{4}']) if 'mitre_techniques' in offense: self.tags.extend(offense['mitre_techniques']) if "categories" in offense: for cat in offense['categories']: self.tags.append(cat) self.defaultObservableDatatype = [ 'autonomous-system', 'domain', 'file', 'filename', 'fqdn', 'hash', 'ip', 'mail', 'mail_subject', 'other', 'process_filename', 'regexp', 'registry', 'uri_path', 'url', 'user-account', 'user-agent' ] self.artifacts = [] for artifact in offense['artifacts']: # Add automation tagging and mitre tagging to observables if len(self.tags_extracted) > 0: artifact['tags'].extend(self.tags_extracted) if 'mitre_tactics' in offense: artifact['tags'].extend(offense['mitre_tactics']) if 'mitre_techniques' in offense: artifact['tags'].extend(offense['mitre_techniques']) if artifact['dataType'] in self.defaultObservableDatatype: self.hiveArtifact = self.theHiveConnector.craftAlertArtifact( dataType=artifact['dataType'], data=artifact['data'], message=artifact['message'], tags=artifact['tags'], tlp=artifact['tlp']) else: artifact['tags'].append('type:' + artifact['dataType']) self.hiveArtifact = self.theHiveConnector.craftAlertArtifact( dataType='other', data=artifact['data'], message=artifact['message'], tags=artifact['tags'], tlp=artifact['tlp']) self.artifacts.append(self.hiveArtifact) # Retrieve the configured case_template self.qradarCaseTemplate = self.cfg.get('QRadar', 'case_template') # Build TheHive alert self.alert = self.theHiveConnector.craftAlert( "{}, {}".format(offense['id'], offense['description']), self.craftAlertDescription(offense), getHiveSeverity(offense), offense['start_time'], self.tags, 2, 'Imported', 'internal', 'QRadar_Offenses', str(offense['id']), self.artifacts, self.qradarCaseTemplate) return self.alert def validateRequest(self, request): if request.is_json: self.content = request.get_json() if 'timerange' in self.content: self.workflowReport = self.allOffense2Alert( self.content['timerange']) if self.workflowReport['success']: return json.dumps(self.workflowReport), 200 else: return json.dumps(self.workflowReport), 500 else: self.logger.error('Missing <timerange> key/value') return json.dumps({ 'sucess': False, 'message': "timerange key missing in request" }), 500 else: self.logger.error('Not json request') return json.dumps({ 'sucess': False, 'message': "Request didn't contain valid JSON" }), 400 def allOffense2Alert(self, timerange): """ Get all openned offense created within the last <timerange> minutes and creates alerts for them in TheHive """ self.logger.info('%s.allOffense2Alert starts', __name__) self.report = dict() self.report['success'] = True self.report['offenses'] = list() try: self.offensesList = self.qradarConnector.getOffenses(timerange) # each offenses in the list is represented as a dict # we enrich this dict with additional details for offense in self.offensesList: self.matched = False # Filter based on regexes in configuration for offense_exclusion_regex in self.cfg.get( 'QRadar', 'offense_exclusion_regexes', fallback=[]): self.logger.debug( "Offense exclusion regex found '{}'. Matching against offense {}" .format(offense_exclusion_regex, offense['id'])) self.regex = re.compile(offense_exclusion_regex) if self.regex.match(offense['description']): self.logger.debug( "Found exclusion match for offense {} and regex {}" .format(offense['id'], offense_exclusion_regex)) self.matched = True if self.matched: continue # Prepare new alert self.offense_report = dict() self.logger.debug("offense: %s" % offense) self.logger.info("Enriching offense...") self.enrichedOffense = self.enrichOffense(offense) self.logger.debug("Enriched offense: %s" % self.enrichedOffense) self.theHiveAlert = self.qradarOffenseToHiveAlert( self.enrichedOffense) # searching if the offense has already been converted to alert self.query = dict() self.query['sourceRef'] = str(offense['id']) self.logger.info('Looking for offense %s in TheHive alerts', str(offense['id'])) self.results = self.theHiveConnector.findAlert(self.query) if len(self.results) == 0: self.logger.info( 'Offense %s not found in TheHive alerts, creating it', str(offense['id'])) try: self.theHiveEsAlertId = self.theHiveConnector.createAlert( self.theHiveAlert)['id'] self.offense_report[ 'raised_alert_id'] = self.theHiveEsAlertId self.offense_report['qradar_offense_id'] = offense[ 'id'] self.offense_report['success'] = True except Exception as e: self.logger.error('%s.allOffense2Alert failed', __name__, exc_info=True) self.offense_report['success'] = False if isinstance(e, ValueError): self.errorMessage = json.loads(str(e))['message'] self.offense_report['message'] = self.errorMessage else: self.offense_report['message'] = str( e) + ": Couldn't raise alert in TheHive" self.offense_report['offense_id'] = offense['id'] # Set overall success if any fails self.report['success'] = False self.report['offenses'].append(self.offense_report) else: self.logger.info( 'Offense %s already imported as alert, checking for updates', str(offense['id'])) self.alert_found = self.results[0] # Check if alert is already created, but needs updating if self.check_if_updated(self.alert_found, vars(self.theHiveAlert)): self.logger.info( "Found changes for %s, updating alert" % self.alert_found['id']) # update alert self.theHiveConnector.updateAlert( self.alert_found['id'], self.theHiveAlert, fields=["tags", "artifacts"]) self.offense_report[ 'updated_alert_id'] = self.alert_found['id'] self.offense_report['qradar_offense_id'] = offense[ 'id'] self.offense_report['success'] = True else: self.logger.info("No changes found for %s" % self.alert_found['id']) continue ########################################################## except Exception as e: self.logger.error( 'Failed to create alert from QRadar offense (retrieving offenses failed)', exc_info=True) self.report['success'] = False self.report[ 'message'] = "%s: Failed to create alert from offense" % str(e) return self.report def craftAlertDescription(self, offense): """ From the offense metadata, crafts a nice description in markdown for TheHive """ self.logger.debug('craftAlertDescription starts') # Start empty self.description = "" # Add url to Offense self.qradar_ip = self.cfg.get('QRadar', 'server') self.url = ( '[%s](https://%s/console/qradar/jsp/QRadar.jsp?appName=Sem&pageId=OffenseSummary&summaryId=%s)' % (str(offense['id']), self.qradar_ip, str(offense['id']))) self.description += '#### Offense: \n - ' + self.url + '\n\n' # Format associated rules self.rule_names_formatted = "#### Rules triggered: \n" self.rules = offense['rules'] if len(self.rules) > 0: for rule in self.rules: if 'name' in rule: self.rule_names_formatted += "- %s \n" % rule['name'] else: continue # Add rules overview to description self.description += self.rule_names_formatted + '\n\n' # Format associated documentation self.uc_links_formatted = "#### Use Case documentation: \n" if 'use_case_names' in offense and offense['use_case_names']: for uc in offense['use_case_names']: self.uc_links_formatted += "- [%s](%s/%s) \n" % ( uc, self.cfg.get('QRadar', 'kb_url'), uc) # Add associated documentation self.description += self.uc_links_formatted + '\n\n' # Add mitre Tactic information self.mitre_ta_links_formatted = "#### MITRE Tactics: \n" if 'mitre_tactics' in offense and offense['mitre_tactics']: for tactic in offense['mitre_tactics']: self.mitre_ta_links_formatted += "- [%s](%s/%s) \n" % ( tactic, 'https://attack.mitre.org/tactics/', tactic) # Add associated documentation self.description += self.mitre_ta_links_formatted + '\n\n' # Add mitre Technique information self.mitre_t_links_formatted = "#### MITRE Techniques: \n" if 'mitre_techniques' in offense and offense['mitre_techniques']: for technique in offense['mitre_techniques']: self.mitre_t_links_formatted += "- [%s](%s/%s) \n" % ( technique, 'https://attack.mitre.org/techniques/', technique) # Add associated documentation self.description += self.mitre_t_links_formatted + '\n\n' # Add offense details table self.description += ( '#### Summary\n\n' + '| | |\n' + '| ----------------------- | ------------- |\n' + '| **Start Time** | ' + str(self.qradarConnector.formatDate(offense['start_time'])) + ' |\n' + '| **Offense ID** | ' + str(offense['id']) + ' |\n' + '| **Description** | ' + str(offense['description'].replace('\n', '')) + ' |\n' + '| **Offense Type** | ' + str(offense['offense_type_str']) + ' |\n' + '| **Offense Source** | ' + str(offense['offense_source']) + ' |\n' + '| **Destination Network** | ' + str(offense['destination_networks']) + ' |\n' + '| **Source Network** | ' + str(offense['source_network']) + ' |\n\n\n' + '\n\n\n\n') # Add raw payload self.description += '```\n' for log in offense['logs']: self.description += log['utf8_payload'] + '\n' self.description += '```\n\n' return self.description
class Automator(): def __init__(self, webhook, cfg, automation_config, modules): """ Class constructor :return: use case report :rtype: API call """ self.logger = logging.getLogger(__name__) self.logger.info('Initiating Siem Integration') self.cfg = cfg self.app_dir = os.path.dirname(os.path.abspath(__file__)) + "/.." self.automation_config = automation_config self.TheHiveConnector = TheHiveConnector(cfg) self.webhook = webhook self.modules = modules if cfg.getboolean('Automation', 'enable_customer_list', fallback=False): self.logger.info('Loading Customer configuration') # Load optional customer config self.customer_cfg = ConfigParser( converters={ 'list': lambda x: [i.strip() for i in x.split(';')] }) self.confPath = self.app_dir + '/conf/customers.conf' try: self.logger.debug('Loading configuration from %s' % self.confPath) self.customer_cfg.read(self.confPath) self.customers = self.customer_cfg.sections() self.logger.debug('Loaded configuration for %s' % self.customers) except Exception as e: self.logger.error('%s', __name__, exc_info=True) def check_automation(self): self.logger.info( 'Start parsing use cases for the SIEM based alerts/cases') self.ucTaskId = False self.report_action = 'None' if 'tags' in self.webhook.data['object']: self.tags = self.webhook.data['object']['tags'] # Add tagging to webhooks that are missing tags elif 'artifactId' in self.webhook.data[ 'object'] and self.webhook.isCaseArtifactJob: self.logger.debug( 'Found artifact id {} for webhook {}. Retrieving tags from there' .format(self.webhook.data['object']['artifactId'], self.webhook.id)) self.tags = self.TheHiveConnector.getCaseObservable( self.webhook.data['object']['artifactId'])['tags'] else: self.tags = [] self.logger.warning("no tags found for webhook {}".format( self.webhook.id)) self.automation_regexes = self.cfg.get('Automation', 'automation_regexes', fallback=None) if not self.automation_regexes: self.logger.error( "Could not find any regexes to find tags for automation") return self.report_action self.automation_ids = self.automation_config['automation_ids'] # loop through tags to see if there is a use case present for tag in self.tags: for automation_regex in self.automation_regexes: # The tag should match this regex otherwise it is no use case try: tag = re.search(automation_regex, tag).group(0) except Exception: self.logger.debug("Tag: %s is not matching the uc regex" % tag) continue # check if use case that is provided, matches the case if tag in self.automation_ids: self.found_a_id = tag # Try to retrieve the defined actions self.use_case_actions = self.automation_ids[ self.found_a_id]['automation'] # perform actions defined for the use case for action, action_config in self.use_case_actions.items(): # Give automator information regarding the webhook as some actions are limited to the state of the alert/case self.logger.info( 'Found the following action for %s: %s, with task %s' % (self.found_a_id, action, action_config['task'])) # Add support for multiple tasks, loop them 1 by 1 if 'tasks' in action_config: for task in action_config['tasks']: action_config['task'] = task # Run actions through the automator if self.Automate(action_config, self.webhook, self.modules): continue else: self.logger.info( 'Did not find any supported actions with details: task:{} tag:{} action:{}' .format(action_config['task'], self.found_a_id, action_config['task'])) #Run actions through the automator else: if self.Automate(action_config, self.webhook, self.modules): continue else: self.logger.info( 'Fallback: Did not find any supported actions with details: task:{} tag:{} action:{}' .format(action_config['task'], self.found_a_id, action_config['task'])) self.logger.info("Report action snapshot for the tag: {}".format( self.report_action)) return self.report_action def Automate(self, task_config, webhook, modules): # Split the task name on the dot to have a module and a function variable in a list try: task = task_config['task'].split(".") # Should probably also do some matching for words to mitigate some security concerns? module_name = task[0] function_name = task[1] except Exception: self.logger.error( "{} does not seem to be a valid automator task name".format( task), exc_info=True) return try: # Load the Automators class from the module to initialise it automators = modules['automators'][module_name].Automators( self.cfg, self.automation_config) except KeyError as e: self.logger.warning( "Automator module not found: {}".format(module_name), exc_info=True) return False try: automator = getattr(automators, '{}'.format(function_name)) # Run the function for the task and return the results results = automator(task_config, webhook) # Return the results or True if the task was succesful without returning information if results: return results else: return False except KeyError as e: self.logger.warning("Automator task not found for {}: {}".format( module_name, function_name), exc_info=True) return False def enrichAlertDescription(self, alert_id, description, enrichment_key, enrichment_value): th_alert_description = self.TheHiveConnector.getAlert( alert_id)['description'] # Split again, then parse the enrichment table part. After that the two splitted parts can be put together again original_description, enrichment_table = retrieveSplittedDescription( description) # Primarily check if the split action worked. If the variables are the same, then the check if the key is found and does not contain the value already if enrichment_table and self.fetchValueFromMDTable( enrichment_table, enrichment_key) != enrichment_value: regex_end_of_table = ' \|\\n\\n\\n' end_of_table = ' |\n\n\n' replacement_description = '|\n | **%s** | %s %s' % ( enrichment_key, enrichment_value, end_of_table) alert_description = re.sub(regex_end_of_table, replacement_description, th_alert_description) enriched = True # Concat enrichment table with rest of description # Update Alert with the new description field updated_alert = Alert updated_alert.description = alert_description self.TheHiveConnector.updateAlert(alert_id, updated_alert, ["description"])