class Automation(): def __init__(self, webhook, cfg): self.logger = logger self.logger.info('Initiating RD Automation') self.TheHiveConnector = TheHiveConnector(cfg) self.webhook = webhook self.cfg = cfg self.report_action = report_action self.RDConnector = RDConnector(cfg) def parse_hooks(self): self.logger.info(f'{__name__}.parse_hooks starts') # Only continue if the right webhook is triggered if self.webhook.isResponsibleDisclosureAlertImported(): pass else: return False try: # Define variables and actions based on certain webhook types self.case_id = self.webhook.data['object']['case'] #parse Mgs ID field from the webhook self.email_id = re.search(r"Msg ID[\s\S]+?\|\s+(\S+)\s+\|", str(self.webhook.data['object'])) self.logger.debug(f"regex match {self.email_id.group(1)}") #get all the attachments and upload to the hive observables attachment_data = self.RDConnector.listAttachment( self.cfg.get('ResponsibleDisclosure', 'email_address'), self.email_id.group(1)) all_attachments = [] if attachment_data: for att in attachment_data: try: file_name = self.RDConnector.downloadAttachments( att['name'], att['attachment_id'], att['isInline'], att['contentType']) all_attachments.append(file_name) self.TheHiveConnector.addFileObservable( self.case_id, file_name, "email attachment") self.logger.info( f"Observable: {file_name} has been updated to Case: {self.case_id}" ) except Exception as e: self.logger.error(e, exc_info=True) continue finally: os.remove(file_name) self.logger.debug( f"File: {file_name} has been removed") return True except Exception as e: self.logger.error(e, exc_info=True) return False
def connectEws(): logger = logging.getLogger(__name__) logger.info('%s.connectEws starts', __name__) report = dict() report['success'] = bool() try: cfg = getConf() ewsConnector = EwsConnector(cfg) folder_name = cfg.get('EWS', 'folder_name') unread = ewsConnector.scan(folder_name) TheHiveConnector = TheHiveConnector(cfg) for msg in unread: #type(msg) #<class 'exchangelib.folders.Message'> conversationId = msg.conversation_id.id #searching if case has already been created from the email #conversation esCaseId = TheHiveConnector.searchCaseByDescription(conversationId) if esCaseId is None: #no case previously created from the conversation caseTitle = str(msg.subject) caseDescription = ('```\n' + 'Case created by Synapse\n' + 'conversation_id: "' + str(msg.conversation_id.id) + '"\n' + '```') if msg.categories: assignee = msg.categories[0] else: assignee = 'synapse' case = TheHiveConnector.craftCase(caseTitle, caseDescription) createdCase = TheHiveConnector.createCase(case) caseUpdated = TheHiveConnector.assignCase( createdCase, assignee) commTask = TheHiveConnector.craftCommTask() esCaseId = caseUpdated.id commTaskId = TheHiveConnector.createTask(esCaseId, commTask) else: #case previously created from the conversation commTaskId = TheHiveConnector.getTaskIdByTitle( esCaseId, 'Communication') if commTaskId != None: pass else: #case already exists but no Communication task found #creating comm task commTask = TheHiveConnector.craftCommTask() commTaskId = TheHiveConnector.createTask( esCaseId, commTask) fullBody = getEmailBody(msg) taskLog = TheHiveConnector.craftTaskLog(fullBody) createdTaskLogId = TheHiveConnector.addTaskLog(commTaskId, taskLog) readMsg = ewsConnector.markAsRead(msg) for attachmentLvl1 in msg.attachments: #uploading the attachment as file observable #is the attachment is a .msg, the eml version #of the file is uploaded tempAttachment = TempAttachment(attachmentLvl1) if not tempAttachment.isInline: #adding the attachment only if it is not inline #inline attachments are pictures in the email body tmpFilepath = tempAttachment.writeFile() to = str() for recipient in msg.to_recipients: to = to + recipient.email_address + ' ' comment = 'Attachment from email sent by ' comment += str(msg.author.email_address).lower() comment += ' and received by ' comment += str(to).lower() comment += ' with subject: <' comment += msg.subject comment += '>' TheHiveConnector.addFileObservable(esCaseId, tmpFilepath, comment) if tempAttachment.isEmailAttachment: #if the attachment is an email #attachments of this email are also #uploaded to TheHive for attachmentLvl2 in tempAttachment.attachments: tempAttachmentLvl2 = TempAttachment(attachmentLvl2) tmpFilepath = tempAttachmentLvl2.writeFile() comment = 'Attachment from the email attached' TheHiveConnector.addFileObservable( esCaseId, tmpFilepath, comment) report['success'] = True return report except Exception as e: logger.error('Failed to create case from email', exc_info=True) report['success'] = False return report
class Integration(Main): def __init__(self): super().__init__() self.RDConnector = RDConnector(self.cfg) self.TheHiveConnector = TheHiveConnector(self.cfg) def validateRequest(self, request): workflowReport = self.connectRD() if workflowReport['success']: return json.dumps(workflowReport), 200 else: return json.dumps(workflowReport), 500 def connectRD(self): self.logger.info('%s.connectResponsibleDisclosure starts', __name__) report = dict() report['success'] = bool() # Setup Tags self.tags = ['Responsible disclosure', 'Synapse'] tracker_file = "./modules/ResponsibleDisclosure/email_tracker" link_to_load = "" if os.path.exists(tracker_file): self.logger.debug("Reading from the tracker file...") with open(tracker_file, "r") as tracker: link_to_load = tracker.read() if not link_to_load: link_to_load = self.cfg.get('ResponsibleDisclosure', 'list_endpoint') emails, new_link = self.RDConnector.scan(link_to_load) try: for email in emails: try: if ('@removed' in email) or [email["from"]["emailAddress"]["address"]] in self.cfg.get('ResponsibleDisclosure', 'excluded_senders'): continue self.logger.debug("Found unread E-mail with id: {}".format(email['id'])) # Get the conversation id from the email CID = email["conversationId"] # Conversation id hash will be used as a unique identifier for the alert CIDHash = hashlib.md5(CID.encode()).hexdigest() email_date = datetime.strptime(email["receivedDateTime"], "%Y-%m-%dT%H:%M:%SZ") epoch_email_date = email_date.timestamp() * 1000 alertTitle = "Responsible Disclosure - {}".format(email["subject"]) alertDescription = self.createDescription(email) # Moving the email from Inbox to the new folder defined by variable to_move_folder in synapse.conf # Disabled temporarily # self.RDConnector.moveToFolder(self.cfg.get('ResponsibleDisclosure', 'email_address'), email['id'], self.cfg.get('ResponsibleDisclosure', 'to_move_folder')) # Get all the attachments and upload to the hive observables attachment_data = self.RDConnector.listAttachment(self.cfg.get('ResponsibleDisclosure', 'email_address'), email['id']) all_artifacts = [] all_attachments = [] if attachment_data: for att in attachment_data: file_name = self.RDConnector.downloadAttachments(att['name'], att['attachment_id'], att['isInline'], att['contentType']) all_attachments.append(file_name) self.af = AlertArtifact(dataType='file', data=file_name, tlp=2, tags=['Responsible disclosure', 'Synapse'], ioc=True) all_artifacts.append(self.af) # Create the alert in thehive alert = self.TheHiveConnector.craftAlert( alertTitle, alertDescription, 1, epoch_email_date, self.tags, 2, "New", "internal", "ResponsibleDisclosure", CIDHash, all_artifacts, self.cfg.get('ResponsibleDisclosure', 'case_template')) # Check if the alert was created successfully query = dict() query['sourceRef'] = str(CIDHash) # Look up if any existing alert in theHive alert_results = self.TheHiveConnector.findAlert(query) # If no alerts are found for corresponding CIDHASH, create a new alert if len(alert_results) == 0: createdAlert = self.TheHiveConnector.createAlert(alert) # automatish antwoord to the original email sender from the responsible disclosure emailaddress autoreply_subject_name = "RE: {}".format(email["subject"]) self.RDConnector.sendAutoReply("*****@*****.**", email["from"]["emailAddress"]["address"], self.cfg.get('ResponsibleDisclosure', 'email_body_filepath'), autoreply_subject_name) # If alert is found update the alert or it may have been migrated to case so update the case if len(alert_results) > 0: alert_found = alert_results[0] # Check if alert is promoted to a case if 'case' in alert_found: case_found = self.TheHiveConnector.getCase(alert_found['case']) # Create a case model self.updated_case = Case # Update the case with new description # What if the email body is empty for new email, then use the old description self.updated_case.description = case_found['description'] + "\n\n" + alertDescription self.updated_case.id = alert_found['case'] self.TheHiveConnector.updateCase(self.updated_case, ["description"]) self.logger.info("updated the description of the case with id: {}".format(alert_found['case'])) # Check if there new observables available if all_attachments: for att in all_attachments: try: self.TheHiveConnector.addFileObservable(alert_found['case'], att, "email attachment") except Exception as e: self.logger.error(f"Encountered an error while creating a new file based observable: {e}", exc_info=True) continue # Else it means there is no corresponding case so update the alert else: # create an alert model self.updated_alert = Alert # Update the alert with new description # What if the email body is empty for new email, then use the old description self.updated_alert.description = alert_found['description'] + "\n\n" + alertDescription self.TheHiveConnector.updateAlert(alert_found['id'], self.updated_alert, ["description"]) self.logger.info("updated the description of the alert with id: {}".format(alert_found['id'])) except Exception as e: self.logger.error(e, exc_info=True) continue if all_attachments: for att in all_attachments: os.remove(att) # Write the delta link to the tracker with open(tracker_file, "w+") as tracker: tracker.write(new_link) report['success'] = True return report except Exception as e: self.logger.error(e) self.logger.error('Connection failure', exc_info=True) report['success'] = False return report def createDescription(self, email): email_body = email['body']['content'] subject = email["subject"] # Get the conversation id from the email CID = email["conversationId"] # Conversation id hash will be used as a unique identifier for the alert CIDHash = hashlib.md5(CID.encode()).hexdigest() # Parse all the URLs and add them to a field in the description table urls_list = re.findall(r'\<(https?://[\S]+?)\>', email_body) # " " is ascii for next line urls_str = ' '.join(str(x) for x in urls_list) from_e = email["from"]["emailAddress"]["address"] to_e = "N/A" if email["toRecipients"]: to_e = email["toRecipients"][0]["emailAddress"]["address"] OriginatingIP = "N/A" for header in email['internetMessageHeaders']: if header['name'] == 'X-Originating-IP': # Formatting the ip value, bydefault it comesup like [x.x.x.x] OriginatingIP = (header['value'][1:-1]) # putting together the markdown table temp_fullbody = [] temp_fullbody.append("| | |") temp_fullbody.append("|:-----|:-----|") temp_fullbody.append("| " + "**" + "Subject" + "**" + " | " + subject + " |") temp_fullbody.append("| " + "**" + "Sender" + "**" + " | " + from_e + " |") temp_fullbody.append("| " + "**" + "Recipient" + "**" + " | " + to_e + " |") temp_fullbody.append("| " + "**" + "Originating IP" + "**" + " | " + OriginatingIP + " |") temp_fullbody.append("| " + "**" + "Received at" + "**" + " | " + email["receivedDateTime"] + " |") temp_fullbody.append("| " + "**" + "URL(s) in email" + "**" + " | " + urls_str + " |") temp_fullbody.append("| " + "**" + "Msg ID" + "**" + " | " + email['id'] + " |") temp_fullbody.append("**" + "Email body" + "**") temp_fullbody.append("```") temp_fullbody.append(email_body) temp_fullbody.append("```") alertDescription = '\r\n'.join(str(x) for x in temp_fullbody) return alertDescription