class Automation():
def __init__(self, webhook, cfg):
self.logger = logger
self.logger.info('Initiating RD Automation')
self.TheHiveConnector = TheHiveConnector(cfg)
self.webhook = webhook
self.cfg = cfg
self.report_action = report_action
self.RDConnector = RDConnector(cfg)
def parse_hooks(self):
self.logger.info(f'{__name__}.parse_hooks starts')
# Only continue if the right webhook is triggered
if self.webhook.isResponsibleDisclosureAlertImported():
pass
else:
return False
try:
# Define variables and actions based on certain webhook types
self.case_id = self.webhook.data['object']['case']
#parse Mgs ID field from the webhook
self.email_id = re.search(r"Msg ID[\s\S]+?\|\s+(\S+)\s+\|",
str(self.webhook.data['object']))
self.logger.debug(f"regex match {self.email_id.group(1)}")
#get all the attachments and upload to the hive observables
attachment_data = self.RDConnector.listAttachment(
self.cfg.get('ResponsibleDisclosure', 'email_address'),
self.email_id.group(1))
all_attachments = []
if attachment_data:
for att in attachment_data:
try:
file_name = self.RDConnector.downloadAttachments(
att['name'], att['attachment_id'], att['isInline'],
att['contentType'])
all_attachments.append(file_name)
self.TheHiveConnector.addFileObservable(
self.case_id, file_name, "email attachment")
self.logger.info(
f"Observable: {file_name} has been updated to Case: {self.case_id}"
)
except Exception as e:
self.logger.error(e, exc_info=True)
continue
finally:
os.remove(file_name)
self.logger.debug(
f"File: {file_name} has been removed")
return True
except Exception as e:
self.logger.error(e, exc_info=True)
return False
def connectEws():
logger = logging.getLogger(__name__)
logger.info('%s.connectEws starts', __name__)
report = dict()
report['success'] = bool()
try:
cfg = getConf()
ewsConnector = EwsConnector(cfg)
folder_name = cfg.get('EWS', 'folder_name')
unread = ewsConnector.scan(folder_name)
TheHiveConnector = TheHiveConnector(cfg)
for msg in unread:
#type(msg)
#<class 'exchangelib.folders.Message'>
conversationId = msg.conversation_id.id
#searching if case has already been created from the email
#conversation
esCaseId = TheHiveConnector.searchCaseByDescription(conversationId)
if esCaseId is None:
#no case previously created from the conversation
caseTitle = str(msg.subject)
caseDescription = ('```\n' + 'Case created by Synapse\n' +
'conversation_id: "' +
str(msg.conversation_id.id) + '"\n' + '```')
if msg.categories:
assignee = msg.categories[0]
else:
assignee = 'synapse'
case = TheHiveConnector.craftCase(caseTitle, caseDescription)
createdCase = TheHiveConnector.createCase(case)
caseUpdated = TheHiveConnector.assignCase(
createdCase, assignee)
commTask = TheHiveConnector.craftCommTask()
esCaseId = caseUpdated.id
commTaskId = TheHiveConnector.createTask(esCaseId, commTask)
else:
#case previously created from the conversation
commTaskId = TheHiveConnector.getTaskIdByTitle(
esCaseId, 'Communication')
if commTaskId != None:
pass
else:
#case already exists but no Communication task found
#creating comm task
commTask = TheHiveConnector.craftCommTask()
commTaskId = TheHiveConnector.createTask(
esCaseId, commTask)
fullBody = getEmailBody(msg)
taskLog = TheHiveConnector.craftTaskLog(fullBody)
createdTaskLogId = TheHiveConnector.addTaskLog(commTaskId, taskLog)
readMsg = ewsConnector.markAsRead(msg)
for attachmentLvl1 in msg.attachments:
#uploading the attachment as file observable
#is the attachment is a .msg, the eml version
#of the file is uploaded
tempAttachment = TempAttachment(attachmentLvl1)
if not tempAttachment.isInline:
#adding the attachment only if it is not inline
#inline attachments are pictures in the email body
tmpFilepath = tempAttachment.writeFile()
to = str()
for recipient in msg.to_recipients:
to = to + recipient.email_address + ' '
comment = 'Attachment from email sent by '
comment += str(msg.author.email_address).lower()
comment += ' and received by '
comment += str(to).lower()
comment += ' with subject: <'
comment += msg.subject
comment += '>'
TheHiveConnector.addFileObservable(esCaseId, tmpFilepath,
comment)
if tempAttachment.isEmailAttachment:
#if the attachment is an email
#attachments of this email are also
#uploaded to TheHive
for attachmentLvl2 in tempAttachment.attachments:
tempAttachmentLvl2 = TempAttachment(attachmentLvl2)
tmpFilepath = tempAttachmentLvl2.writeFile()
comment = 'Attachment from the email attached'
TheHiveConnector.addFileObservable(
esCaseId, tmpFilepath, comment)
report['success'] = True
return report
except Exception as e:
logger.error('Failed to create case from email', exc_info=True)
report['success'] = False
return report
class Integration(Main):
def __init__(self):
super().__init__()
self.RDConnector = RDConnector(self.cfg)
self.TheHiveConnector = TheHiveConnector(self.cfg)
def validateRequest(self, request):
workflowReport = self.connectRD()
if workflowReport['success']:
return json.dumps(workflowReport), 200
else:
return json.dumps(workflowReport), 500
def connectRD(self):
self.logger.info('%s.connectResponsibleDisclosure starts', __name__)
report = dict()
report['success'] = bool()
# Setup Tags
self.tags = ['Responsible disclosure', 'Synapse']
tracker_file = "./modules/ResponsibleDisclosure/email_tracker"
link_to_load = ""
if os.path.exists(tracker_file):
self.logger.debug("Reading from the tracker file...")
with open(tracker_file, "r") as tracker:
link_to_load = tracker.read()
if not link_to_load:
link_to_load = self.cfg.get('ResponsibleDisclosure', 'list_endpoint')
emails, new_link = self.RDConnector.scan(link_to_load)
try:
for email in emails:
try:
if ('@removed' in email) or [email["from"]["emailAddress"]["address"]] in self.cfg.get('ResponsibleDisclosure', 'excluded_senders'):
continue
self.logger.debug("Found unread E-mail with id: {}".format(email['id']))
# Get the conversation id from the email
CID = email["conversationId"]
# Conversation id hash will be used as a unique identifier for the alert
CIDHash = hashlib.md5(CID.encode()).hexdigest()
email_date = datetime.strptime(email["receivedDateTime"], "%Y-%m-%dT%H:%M:%SZ")
epoch_email_date = email_date.timestamp() * 1000
alertTitle = "Responsible Disclosure - {}".format(email["subject"])
alertDescription = self.createDescription(email)
# Moving the email from Inbox to the new folder defined by variable to_move_folder in synapse.conf
# Disabled temporarily
# self.RDConnector.moveToFolder(self.cfg.get('ResponsibleDisclosure', 'email_address'), email['id'], self.cfg.get('ResponsibleDisclosure', 'to_move_folder'))
# Get all the attachments and upload to the hive observables
attachment_data = self.RDConnector.listAttachment(self.cfg.get('ResponsibleDisclosure', 'email_address'), email['id'])
all_artifacts = []
all_attachments = []
if attachment_data:
for att in attachment_data:
file_name = self.RDConnector.downloadAttachments(att['name'], att['attachment_id'], att['isInline'], att['contentType'])
all_attachments.append(file_name)
self.af = AlertArtifact(dataType='file', data=file_name, tlp=2, tags=['Responsible disclosure', 'Synapse'], ioc=True)
all_artifacts.append(self.af)
# Create the alert in thehive
alert = self.TheHiveConnector.craftAlert(
alertTitle,
alertDescription,
1,
epoch_email_date,
self.tags, 2,
"New",
"internal",
"ResponsibleDisclosure",
CIDHash,
all_artifacts,
self.cfg.get('ResponsibleDisclosure', 'case_template'))
# Check if the alert was created successfully
query = dict()
query['sourceRef'] = str(CIDHash)
# Look up if any existing alert in theHive
alert_results = self.TheHiveConnector.findAlert(query)
# If no alerts are found for corresponding CIDHASH, create a new alert
if len(alert_results) == 0:
createdAlert = self.TheHiveConnector.createAlert(alert)
# automatish antwoord to the original email sender from the responsible disclosure emailaddress
autoreply_subject_name = "RE: {}".format(email["subject"])
self.RDConnector.sendAutoReply("*****@*****.**", email["from"]["emailAddress"]["address"], self.cfg.get('ResponsibleDisclosure', 'email_body_filepath'), autoreply_subject_name)
# If alert is found update the alert or it may have been migrated to case so update the case
if len(alert_results) > 0:
alert_found = alert_results[0]
# Check if alert is promoted to a case
if 'case' in alert_found:
case_found = self.TheHiveConnector.getCase(alert_found['case'])
# Create a case model
self.updated_case = Case
# Update the case with new description
# What if the email body is empty for new email, then use the old description
self.updated_case.description = case_found['description'] + "\n\n" + alertDescription
self.updated_case.id = alert_found['case']
self.TheHiveConnector.updateCase(self.updated_case, ["description"])
self.logger.info("updated the description of the case with id: {}".format(alert_found['case']))
# Check if there new observables available
if all_attachments:
for att in all_attachments:
try:
self.TheHiveConnector.addFileObservable(alert_found['case'], att, "email attachment")
except Exception as e:
self.logger.error(f"Encountered an error while creating a new file based observable: {e}", exc_info=True)
continue
# Else it means there is no corresponding case so update the alert
else:
# create an alert model
self.updated_alert = Alert
# Update the alert with new description
# What if the email body is empty for new email, then use the old description
self.updated_alert.description = alert_found['description'] + "\n\n" + alertDescription
self.TheHiveConnector.updateAlert(alert_found['id'], self.updated_alert, ["description"])
self.logger.info("updated the description of the alert with id: {}".format(alert_found['id']))
except Exception as e:
self.logger.error(e, exc_info=True)
continue
if all_attachments:
for att in all_attachments:
os.remove(att)
# Write the delta link to the tracker
with open(tracker_file, "w+") as tracker:
tracker.write(new_link)
report['success'] = True
return report
except Exception as e:
self.logger.error(e)
self.logger.error('Connection failure', exc_info=True)
report['success'] = False
return report
def createDescription(self, email):
email_body = email['body']['content']
subject = email["subject"]
# Get the conversation id from the email
CID = email["conversationId"]
# Conversation id hash will be used as a unique identifier for the alert
CIDHash = hashlib.md5(CID.encode()).hexdigest()
# Parse all the URLs and add them to a field in the description table
urls_list = re.findall(r'\<(https?://[\S]+?)\>', email_body)
# " " is ascii for next line
urls_str = ' '.join(str(x) for x in urls_list)
from_e = email["from"]["emailAddress"]["address"]
to_e = "N/A"
if email["toRecipients"]:
to_e = email["toRecipients"][0]["emailAddress"]["address"]
OriginatingIP = "N/A"
for header in email['internetMessageHeaders']:
if header['name'] == 'X-Originating-IP':
# Formatting the ip value, bydefault it comesup like [x.x.x.x]
OriginatingIP = (header['value'][1:-1])
# putting together the markdown table
temp_fullbody = []
temp_fullbody.append("| | |")
temp_fullbody.append("|:-----|:-----|")
temp_fullbody.append("| " + "**" + "Subject" + "**" + " | " + subject + " |")
temp_fullbody.append("| " + "**" + "Sender" + "**" + " | " + from_e + " |")
temp_fullbody.append("| " + "**" + "Recipient" + "**" + " | " + to_e + " |")
temp_fullbody.append("| " + "**" + "Originating IP" + "**" + " | " + OriginatingIP + " |")
temp_fullbody.append("| " + "**" + "Received at" + "**" + " | " + email["receivedDateTime"] + " |")
temp_fullbody.append("| " + "**" + "URL(s) in email" + "**" + " | " + urls_str + " |")
temp_fullbody.append("| " + "**" + "Msg ID" + "**" + " | " + email['id'] + " |")
temp_fullbody.append("**" + "Email body" + "**")
temp_fullbody.append("```")
temp_fullbody.append(email_body)
temp_fullbody.append("```")
alertDescription = '\r\n'.join(str(x) for x in temp_fullbody)
return alertDescription