def manageWebhook(webhookData):
"""
Filter webhooks received from TheHive and initiate actions like:
- closing offense in QRadar
"""
logger = logging.getLogger(__name__)
logger.info('%s.ManageWebhook starts', __name__)
report = dict()
cfg = getConf()
webhook = Webhook(webhookData, cfg)
actuator = Actuator(cfg)
#we are only interrested in update webhook at the moment
if webhook.isUpdate():
if webhook.isQRadarAlertMarkedAsRead():
actuator.closeOffense(webhook.offenseId)
report['action'] = 'closeOffense'
elif webhook.isClosedQRadarCase():
actuator.closeOffense(webhook.offenseId)
report['action'] = 'closeOffense'
else:
#is not update or not a QRadar alert marked as read or not a closed QRadar case
report['action'] = 'None'
report['success'] = True
return report
def closingQradarOffense(offense_id, note_text, resol_status):
logger = logging.getLogger(__name__)
logger.info('%s.closingQradarOffense starts', __name__)
report = dict()
report['success'] = bool()
try:
cfg = getConf()
qradarConnector = QRadarConnector(cfg)
#Add closing summary as a note to the offense
response_note = qradarConnector.addNote(offense_id, note_text)
#Determine closing reason
if resol_status is None or resol_status == "Indeterminate":
closing_reason_id = '1' #Non-Issue
elif resol_status == "FalsePositive":
closing_reason_id = '2' #False Positive
else:
closing_reason_id = '3' #Policy Violation
response_closing = qradarConnector.closeOffense(
offense_id, closing_reason_id)
report['success'] = True
return report
except Exception as e:
logger.error('Failed to close QRadar offense', exc_info=True)
report['success'] = False
return report
def craftAlertDescription(offense):
"""
From the offense metadata, crafts a nice description in markdown
for TheHive
"""
logger = logging.getLogger(__name__)
logger.info('craftAlertDescription starts')
cfg = getConf()
QRadarIp = cfg.get('QRadar', 'server')
url = ('https://' + QRadarIp + '/console/qradar/jsp/QRadar.jsp?' +
'appName=Sem&pageId=OffenseSummary&summaryId=' + str(offense['id']))
description = (
'## Summary\n\n' + '| | |\n' +
'| ----------------------- | ------------- |\n' +
'| **Offense ID** | ' + str(offense['id']) + ' |\n' +
'| **Description** | ' +
str(offense['description'].replace('\n', '')) + ' |\n' +
'| **Offense Type** | ' + str(offense['offense_type_str']) +
' |\n' + '| **Offense Source** | ' +
str(offense['offense_source']) + ' |\n' +
'| **Destination Network** | ' + str(offense['destination_networks']) +
' |\n' + '| **Source Network** | ' +
str(offense['source_network']) + ' |\n\n\n' + '\n\n\n\n```\n')
for log in offense['logs']:
description += log['utf8_payload'] + '\n'
description += '```\n\n' + url
return description
def manageWebhook(webhookData):
"""
Filter webhooks received from TheHive and initiate actions like:
- closing offense in QRadar
"""
logger = logging.getLogger(__name__)
logger.info('%s.ManageWebhook starts', __name__)
report = dict()
cfg = getConf()
logger.info('---------------------------------------')
webhook = Webhook(webhookData, cfg)
mailSender = SendMail(cfg)
mailSender.processWebhook(webhookData, cfg)
actuator = Actuator(cfg)
print(webhookData)
#we are only interrested in update webhook at the moment
if webhook.isUpdate():
severity = 0
try:
severity = webhookData['object']['severity']
print(severity)
except Exception as e:
self.logger.error('Ne postoji severity, ostaje 0')
report['action'] = 'Update'
if webhook.isQRadarAlertMarkedAsRead():
actuator.closeOffense(webhook.offenseId, severity)
report['action'] = 'closeOffense'
loger.info("is readed")
elif webhook.isClosedQRadarCase():
actuator.closeOffense(webhook.offenseId, severity)
report['action'] = 'closeOffense'
loger.info("is closed")
else:
#is not update or not a QRadar alert marked as read or not a closed QRadar case
report['action'] = 'None'
report['success'] = True
return report
def phishingAlert():
report = dict()
report['success'] = bool()
tempAttachment = None
cfg = getConf()
ewsConnector = EwsConnector(cfg)
folder_name = cfg.get('EWS', 'folder_name')
unread = ewsConnector.scan(folder_name)
theHiveConnector = TheHiveConnector(cfg)
for email in unread:
conversationId = email.conversation_id.id
alertTitle = str(email.subject)
alertDescription = ('```\n' + 'Alert created by Synapse\n' +
'conversation_id: "' +
str(email.conversation_id.id) + '"\n' + '```')
alertArtifacts = []
alertTags = ['CAT 7']
for msg in email.attachments:
try:
#print(type(msg))
q = dict()
q['sourceRef'] = str(conversationId)
esAlertId = theHiveConnector.findAlert(q)
tempAttachment = TempAttachment(msg)
if not tempAttachment.isInline:
#print('here')
tmpFilepath = tempAttachment.writeFile()
with open(tmpFilepath, 'rb') as fhdl:
raw_email = fhdl.read()
parsed_eml = eml_parser.eml_parser.decode_email_b(
raw_email)
#print(parsed_eml['header']['header']['to'])
#print(json.dumps(parsed_eml, default=json_serial, indent=4, sort_keys=True))
alertArtifacts.append(
theHiveConnector.craftAlertArtifact(
dataType='file',
message="Phishing Email",
data=tmpFilepath,
tags=['Synapse']))
alertArtifacts.append(
theHiveConnector.craftAlertArtifact(
dataType='other',
message="Message Id",
data=parsed_eml['header']['header']['message-id']
[0],
tags=['Synapse']))
for i in parsed_eml['header']['received_ip']:
alertArtifacts.append(
theHiveConnector.craftAlertArtifact(
dataType='ip',
message="Source IP",
data=i,
tags=['Synapse']))
alertArtifacts.append(
theHiveConnector.craftAlertArtifact(
dataType='mail_subject',
message="Phishing Email Subject",
data=parsed_eml['header']['subject'],
tags=['Synapse']))
for i in parsed_eml['header']['to']:
alertArtifacts.append(
theHiveConnector.craftAlertArtifact(
dataType='mail',
message="Recipients",
data=i,
tags=['Synapse']))
for i in parsed_eml['header']['header']['return-path']:
alertArtifacts.append(
theHiveConnector.craftAlertArtifact(
dataType='mail',
message="Return Path",
data=i,
tags=['Synapse']))
if 'x-originating-ip' in parsed_eml['header']['header']:
alertArtifacts.append(
theHiveConnector.craftAlertArtifact(
dataType='mail',
message="Origin IP",
data=parsed_eml['header']['header']
['x-originating-ip'],
tags=['Synapse']))
alert = theHiveConnector.craftAlert(
alertTitle,
alertDescription,
severity=2,
tlp=2,
status="New",
date=(int(time.time() * 1000)),
tags=alertTags,
type="Phishing",
source="Phishing Mailbox",
sourceRef=email.conversation_id.id,
artifacts=alertArtifacts,
caseTemplate="Category 7 - Phishing")
theHiveEsAlertId = theHiveConnector.createAlert(
alert)['id']
except Exception as e:
#msg_obj = msg_parser.msg_parser.Message(msg)
#print(msg_obj.get_message_as_json())
#msg_properties_dict = msg_obj.get_properties()
print('Failed to create alert from attachment')
readMsg = ewsConnector.markAsRead(email)
def connectEws():
report = dict()
report['success'] = bool()
clients = None
with open('/opt/Synapse/conf/mailboxes.json') as f:
clients = json.load(f)
mailboxes = clients['mailboxes']
cfg = getConf()
theHiveConnector = TheHiveConnector(cfg)
for mailbox in mailboxes:
ewsConnector = EwsConnector(username=mailbox['username'],
password=mailbox['password'],
auth_type=mailbox['auth_type'],
server=mailbox['server'],
smtp_address=mailbox['smtp_address'])
unread = ewsConnector.scan(mailbox['folder_name'])
for msg in unread:
#type(msg)
#<class 'exchangelib.folders.Message'>
conversationId = msg.conversation_id.id
origin_mail = msg.attachments[0].item
fullBody = getEmailBody(msg)
alertTitle = "{}; {}".format(
msg.subject, ''.join(
[i if ord(i) < 128 else '' for i in origin_mail.subject]))
alertDescription = ('```\n' + 'Alert created by Synapse\n' +
'conversation_id: "' +
str(msg.conversation_id.id) + '"\n\n' +
'Original mail:\n\n' + fullBody[3:])
# rapporteur
reporter = str(msg.sender.email_address).lower()
# expéditeur
sender = str(origin_mail.sender.email_address).lower()
# date de réception originale et en-tant que spam
rcv_date_timestamp = origin_mail.datetime_received.timestamp(
) * 1000
# date de signalement à la boîte support
rpt_date_timestamp = msg.datetime_received.timestamp() * 1000
customFields = CustomFieldHelper()\
.add_string('reporter', reporter)\
.add_string('sender', sender)\
.add_date('receivedDate', rcv_date_timestamp)\
.add_date('reportedDate', rpt_date_timestamp)\
.add_string('client', mailbox['name'])\
.build()
file_paths = []
artifacts = []
artifacts.append(
AlertArtifact(dataType='mail',
data=reporter,
ignoreSimilarity=True,
tags=['Synapse']))
artifacts.append(
AlertArtifact(dataType='mail',
data=sender,
message="Original sender of the e-mail",
tags=['Synapse', 'Sender']))
artifacts.append(
AlertArtifact(dataType='mail_subject',
data=str(origin_mail.subject),
message="Original subject of the e-mail",
tags=['Synapse']))
attachedFiles = getFileAttachments(msg)
for attached in attachedFiles:
artifacts.append(
AlertArtifact(dataType='file',
data=[attached['path']],
tags=['Attachment', 'Synapse']))
file_paths.append(attached['path'])
alert = Alert(title=alertTitle,
description=alertDescription,
severity=2,
date=rcv_date_timestamp,
tags=[],
tlp=2,
status="New",
type="Phishing",
source="BAL Phishing",
sourceRef="{}-{}".format(
mailbox['short_name'],
str(uuid.uuid4())[24:].upper()),
artifacts=artifacts,
caseTemplate="Suspicious Email Case",
customFields=customFields)
theHiveConnector.createAlert(alert)
ewsConnector.markAsRead(msg)
logger.info("Cleaning temp files ...")
for temp_file in file_paths:
try:
os.remove(temp_file)
except OSError as errRm:
logger.error("'{}' could not be removed.{}".format(
temp_file, errRm))
report['success'] = True
return report
def createQradarAlert():
logger = logging.getLogger(__name__)
logger.info('%s.createQradarAlert starts', __name__)
report = dict()
report['success'] = bool()
try:
cfg = getConf()
qradarConnector = QRadarConnector(cfg)
theHiveConnector = TheHiveConnector(cfg)
#Retrieve QRadar offenses with "OPEN" status
response = qradarConnector.getOffenses()
for offense in response:
#Check if offense is already imported in TheHive
theHive_alert = theHiveConnector.getAlerts({
'sourceRef':
'QR' + str(offense['id'])
}).json()
if theHive_alert == []:
print('QR' + str(offense['id']) + ' not imported')
#Import opened offense in TheHive
##Create a list of AlertArtifact objects
artifact_fields = {
'source_network': ('domain', 'STRING'),
'destination_networks': ('domain', 'STRING_LIST'),
'offense_source': ('ip', 'STRING')
}
artifacts_dict = defaultdict(list)
for field in artifact_fields:
if artifact_fields[field][1] == 'STRING_LIST':
for elmt in offense[field]:
artifacts_dict[artifact_fields[field][0]].append(
elmt)
elif artifact_fields[field][1] == 'STRING':
artifacts_dict[artifact_fields[field][0]].append(
offense[field])
artifacts_list = theHiveConnector.craftAlertArtifact(
artifacts_dict)
print(artifacts_dict)
print(artifacts_list)
##Prepare other fields for an alert
title = "#" + str(
offense['id']) + " QRadar - " + offense['description']
description = ' / '.join(offense['categories'])
if offense['severity'] < 3:
severity = 1
elif offense['severity'] > 6:
severity = 3
else:
severity = 2
date = offense['start_time']
tags = [
'Synapse', 'src:QRadar', 'QRadarID:' + str(offense['id'])
]
sourceRef = 'QR' + str(offense['id'])
##Create Alert object and send it to TheHive
alert = theHiveConnector.craftAlert(title, description,
severity, date, tags,
sourceRef, artifacts_list)
theHiveConnector.createAlert(alert)
else:
print('QR' + str(offense['id']) + ' already imported')
report['success'] = True
return report
except Exception as e:
logger.error('Failed to create alert from QRadar offense',
exc_info=True)
report['success'] = False
return report
def connectEws():
logger = logging.getLogger(__name__)
logger.info('%s.connectEws starts', __name__)
report = dict()
report['success'] = bool()
try:
cfg = getConf()
ewsConnector = EwsConnector(cfg)
folder_name = cfg.get('EWS', 'folder_name')
unread = ewsConnector.scan(folder_name)
theHiveConnector = TheHiveConnector(cfg)
api = TheHiveApi('http://127.0.0.1:9000', API_KEY)
for msg in unread:
#type(msg)
#<class 'exchangelib.folders.Message'>
conversationId = msg.conversation_id.id
#searching if case has already been created from the email
#conversation
esCaseId = theHiveConnector.searchCaseByDescription(conversationId)
if esCaseId is None:
#no case previously created from the conversation
caseTitle = str(msg.subject)
caseDescription = ('```\n' + 'Case created by Synapse\n' +
'conversation_id: "' +
str(msg.conversation_id.id) + '"\n' + '```')
if msg.categories:
assignee = msg.categories[0]
else:
assignee = 'synapse'
case = theHiveConnector.craftCase(caseTitle, caseDescription)
createdCase = theHiveConnector.createCase(case)
caseUpdated = theHiveConnector.assignCase(
createdCase, assignee)
commTask = theHiveConnector.craftCommTask()
esCaseId = caseUpdated.id
commTaskId = theHiveConnector.createTask(esCaseId, commTask)
else:
#case previously created from the conversation
commTaskId = theHiveConnector.getTaskIdByTitle(
esCaseId, 'Communication')
if commTaskId != None:
pass
else:
#case already exists but no Communication task found
#creating comm task
commTask = theHiveConnector.craftCommTask()
commTaskId = theHiveConnector.createTask(
esCaseId, commTask)
fullBody = getEmailBody(msg)
#Scan body message for observables, returns list of observables
observables = searchObservables(fullBody)
taskLog = theHiveConnector.craftTaskLog(fullBody)
createdTaskLogId = theHiveConnector.addTaskLog(commTaskId, taskLog)
readMsg = ewsConnector.markAsRead(msg)
for attachmentLvl1 in msg.attachments:
#uploading the attachment as file observable
#is the attachment is a .msg, the eml version
#of the file is uploaded
tempAttachment = TempAttachment(attachmentLvl1)
if not tempAttachment.isInline:
#adding the attachment only if it is not inline
#inline attachments are pictures in the email body
tmpFilepath = tempAttachment.writeFile()
to = str()
for recipient in msg.to_recipients:
to = to + recipient.email_address + ' '
comment = 'Attachment from email sent by '
comment += str(msg.author.email_address).lower()
comment += ' and received by '
comment += str(to).lower()
comment += ' with subject: <'
comment += msg.subject
comment += '>'
theHiveConnector.addFileObservable(esCaseId, tmpFilepath,
comment)
if tempAttachment.isEmailAttachment:
#if the attachment is an email
#attachments of this email are also
#uploaded to TheHive
for attachmentLvl2 in tempAttachment.attachments:
tempAttachmentLvl2 = TempAttachment(attachmentLvl2)
tmpFilepath = tempAttachmentLvl2.writeFile()
comment = 'Attachment from the email attached'
theHiveConnector.addFileObservable(
esCaseId, tmpFilepath, comment)
#Parse obserables
for o in observables:
if isWhitelisted(o['value']):
print("skipping %s" % o['value'])
else:
observable = CaseObservable(
dataType=o['type'],
data=o['value'],
tlp=2,
ioc=False,
tags=['Synapse'],
message='Found in the email body')
#send observables to case
response = api.create_case_observable(esCaseId, observable)
time.sleep(1)
report['success'] = True
return report
except Exception as e:
logger.error('Failed to create case from email', exc_info=True)
report['success'] = False
return report
def allOffense2Alert(timerange):
"""
Get all openned offense created within the last
<timerange> minutes and creates alerts for them in
TheHive
"""
logger = logging.getLogger(__name__)
logger.info('%s.allOffense2Alert starts', __name__)
report = dict()
report['success'] = True
report['offenses'] = list()
try:
cfg = getConf()
qradarConnector = QRadarConnector(cfg)
theHiveConnector = TheHiveConnector(cfg)
offensesList = qradarConnector.getOffenses(timerange)
#each offenses in the list is represented as a dict
#we enrich this dict with additional details
for offense in offensesList:
#searching if the offense has already been converted to alert
q = dict()
q['sourceRef'] = str(offense['id'])
logger.info('Looking for offense %s in TheHive alerts',
str(offense['id']))
results = theHiveConnector.findAlert(q)
if len(results) == 0:
logger.info(
'Offense %s not found in TheHive alerts, creating it',
str(offense['id']))
offense_report = dict()
enrichedOffense = enrichOffense(qradarConnector, offense)
try:
theHiveAlert = qradarOffenseToHiveAlert(
theHiveConnector, enrichedOffense)
theHiveEsAlertId = theHiveConnector.createAlert(
theHiveAlert)['id']
offense_report['raised_alert_id'] = theHiveEsAlertId
offense_report['qradar_offense_id'] = offense['id']
offense_report['success'] = True
except Exception as e:
logger.error('%s.allOffense2Alert failed',
__name__,
exc_info=True)
offense_report['success'] = False
if isinstance(e, ValueError):
errorMessage = json.loads(str(e))['message']
offense_report['message'] = errorMessage
else:
offense_report['message'] = str(
e) + ": Couldn't raise alert in TheHive"
offense_report['offense_id'] = offense['id']
# Set overall success if any fails
report['success'] = False
report['offenses'].append(offense_report)
else:
logger.info('Offense %s already imported as alert',
str(offense['id']))
except Exception as e:
logger.error(
'Failed to create alert from QRadar offense (retrieving offenses failed)',
exc_info=True)
report['success'] = False
report['message'] = "%s: Failed to create alert from offense" % str(e)
return report
def craftAlertDescription(offense):
"""
From the offense metadata, crafts a nice description in markdown
for TheHive
"""
logger = logging.getLogger(__name__)
logger.info('craftAlertDescription starts')
cfg = getConf()
QRadarIp = cfg.get('QRadar', 'server')
url = "https://{QRadarIp}/console/qradar/jsp/QRadar.jsp?appName=Sem&pageId=OffenseSummary&summaryId={offense_id}"\
.format(QRadarIp=QRadarIp, offense_id=offense['id'])
# Summary
description = (
"| **Summary** | {offense_id} |\n"\
"| :---------------------- | :--------------------: |\n"\
"| **Description** | {description} |\n"\
"| **Domain** | {domain_str} |\n"\
"| **Offense Type** | {offense_type_str} |\n"\
"| **Offense Source** | {offense_source} |\n"\
"| **Destination Network** | {destination_networks} |\n"\
"| **Source Network** | {source_network} |\n"\
"| **Initial Event Count** | {event_count} |\n\n"\
.format(
offense_id=offense['id'],
description=offense['description'].replace('\n', ''),
domain_str=offense['domain_str'],
offense_type_str=offense['offense_type_str'],
offense_source=offense['offense_source'],
destination_networks=", ".join(offense['destination_networks']),
source_network=offense['source_network'],
event_count=offense['event_count']))
# Contributing rules
#description += (
# '---\n' +
# '| **Contributing Rules** |\n' +
# '| :----------------------: |\n')
#for rule in offense['rules']:
# description += '| ' + str(rule) + ' |\n'
# CRE Events
description += ('\n\n' + '---\n' + '| **CRE Event** |\n' +
'| ------------- |\n')
for log in offense['logs']:
description += '| {log} |\n'.format(
log=log['utf8_payload'].replace('\n', ' '))
# First 3 raw logs
#description += '\n\n```\n'
#for log in offense['logs']:
# description += log['utf8_payload'] + '\n'
#description += '```\n\n'
# QRadar URL
description += '\n\n'\
'---\n'\
'### [🔍 Investigate in QRadar]({offense_url})\n'\
'---\n'.format(offense_url=url)
return description
def allOffense2Alert(timerange):
"""
Get all openned offense created within the last
<timerange> minutes and creates alerts for them in
TheHive
"""
logger = logging.getLogger(__name__)
logger.info('%s.allOffense2Alert starts', __name__)
report = dict()
report['success'] = True
report['offenses'] = list()
try:
cfg = getConf()
qradarConnector = QRadarConnector(cfg)
theHiveConnector = TheHiveConnector(cfg)
offensesList = qradarConnector.getOffenses(timerange)
#each offenses in the list is represented as a dict
#we enrich this dict with additional details
for offense in offensesList:
#searching if the offense has already been converted to alert
logger.info('Looking for offense %s in TheHive alerts',
str(offense['id']))
# Update only new Alerts, as Ignored it will be closed on QRadar and should not be updated,
# as Imported we will do a responder to fetch latest info in the case
results = theHiveConnector.findAlert(
Eq("sourceRef", str(offense['id'])))
offense_report = dict()
try:
if len(results) == 0:
logger.info(
'Offense %s not found in TheHive alerts, creating it',
str(offense['id']))
enrichedOffense = enrichOffense(qradarConnector, offense)
theHiveAlert = qradarOffenseToHiveAlert(
theHiveConnector, enrichedOffense)
theHiveEsAlertId = theHiveConnector.createAlert(
theHiveAlert)['id']
offense_report['type'] = "Creation"
offense_report['raised_alert_id'] = theHiveEsAlertId
offense_report['qradar_offense_id'] = offense['id']
offense_report['success'] = True
report['offenses'].append(offense_report)
elif results[0]['status'] not in ['Ignored', 'Imported']:
# update alert if alert is not imported and not dimissed
# will only update 'lastEventCount' and 'lastUpdatedTime' custom fields
logger.info('Updating offense %s', str(offense['id']))
alert = Alert(json=results[0])
cf = CustomFieldHelper()
alert.title = offense['description']
if 'lastEventCount' not in alert.customFields:
alert.customFields['lastEventCount'] = {}
if 'lastUpdated' not in alert.customFields:
alert.customFields['lastUpdated'] = {}
if 'offenseSource' not in alert.customFields:
alert.customFields['offenseSource'] = {}
alert.customFields['lastEventCount']['number'] = offense[
'event_count']
alert.customFields['lastUpdated']['date'] = offense[
'last_updated_time']
alert.customFields['offenseSource']['string'] = offense[
'offense_source'] # updated maybe ?
# should improve TheHiveConnector.updateAlert() rather than using this
updatedAlert = theHiveConnector.theHiveApi.update_alert(
results[0]['id'],
alert,
fields=['customFields', 'title'])
if not updatedAlert.ok:
raise ValueError(json.dumps(updatedAlert.json()))
offense_report['type'] = "Update"
offense_report['updated_alert_id'] = updatedAlert.json(
)['id']
offense_report['qradar_offense_id'] = offense['id']
offense_report['success'] = True
report['offenses'].append(offense_report)
else:
logger.info("Offense already exists")
except Exception as e:
logger.error('%s.allOffense2Alert failed',
__name__,
exc_info=True)
offense_report['success'] = False
if isinstance(e, ValueError):
errorMessage = json.loads(str(e))['message']
offense_report['message'] = errorMessage
else:
offense_report['message'] = str(
e) + ": Couldn't raise alert in TheHive"
offense_report['offense_id'] = offense['id']
# Set overall success if any fails
report['success'] = False
except Exception as e:
logger.error(
'Failed to create alert from QRadar offense (retrieving offenses failed)',
exc_info=True)
report['success'] = False
report['message'] = "%s: Failed to create alert from offense" % str(e)
return report
# description += '| ' + str(rule) + ' |\n'
# CRE Events
description += ('\n\n' + '---\n' + '| **CRE Event** |\n' +
'| ------------- |\n')
for log in offense['logs']:
description += '| {log} |\n'.format(
log=log['utf8_payload'].replace('\n', ' '))
# First 3 raw logs
#description += '\n\n```\n'
#for log in offense['logs']:
# description += log['utf8_payload'] + '\n'
#description += '```\n\n'
# QRadar URL
description += '\n\n'\
'---\n'\
'### [🔍 Investigate in QRadar]({offense_url})\n'\
'---\n'.format(offense_url=url)
return description
if __name__ == '__main__':
#hardcoding timerange as 1 minute when not using the API
#timerange = 1
#offense2Alert(timerange)
cfg = getConf()
QRadarConnector(cfg).getDomainStr(2)
def allNotifs2Alert():
logger = logging.getLogger('workflows.' + __name__)
logger.info('%s.allNotifs2Alert starts', __name__)
result = dict()
result['success'] = bool()
result['message'] = str()
try:
carbonBlack = CBConnector()
# DEBUG PURPOSES ONLY !
#logger.info(str(allNotifications))
conf = getConf()
theHiveConnector = TheHiveConnector(conf)
with open(os.path.join(current_dir, "..", "conf",
"carbonblack.json")) as fd:
organizations = json.load(fd)['orgs']
for org in organizations:
notifications = carbonBlack.getAllNotifications(
org['notifications_profile'], org['alerts_profile'])
for notification in notifications:
#TODO: maybe we should set ALL the variables containing relevant info here to avoid .get() everywhere, btw is .get() actually usefull ?
#TODO: maybe cut a lot of this variable process in a few (or a lot of) functions, juste like "descriptionCrafter" and "artifactCrafter"
# This and the next try...catch is to avoid backslashes '\' in a tag, as it is breaking TheHive sorting mechanism
orgName = org['name']
orgTagName = org['tag-name']
orgShortName = org['short-name']
orgId = org['orgId']
client = org['jira-project']
deviceName = str(notification['deviceInfo']['deviceName'])
summary = str(notification['threatInfo']['summary'])
severity = int(SEVERITIES[int(
notification['threatInfo']['score'])])
date_created = int(notification['eventTime'])
source_ref = "{}-{}".format(
orgShortName,
str(notification['threatInfo']['incidentId']))
sensor_id = str(notification['deviceInfo']['deviceId'])
offense_id = str(notification['threatInfo']['incidentId'])
tags = []
customFields = CustomFieldHelper()\
.add_string('client', client)\
.add_string('sensorID', sensor_id)\
.add_string('hostname', deviceName)\
.build()
artifacts = artifactCrafter(notification, theHiveConnector,
tags)
artifacts.append(
AlertArtifact(dataType='carbon_black_alert_id',
data=offense_id,
message="ID of alert in Carbon Black",
tags=[offense_id],
ignoreSimilarity=True))
alert = theHiveConnector.craftAlert(
title=summary,
description=descriptionCrafter(notification, orgName,
orgId),
severity=severity,
date=date_created,
tags=tags,
tlp=2,
status="New",
type='EDR',
source='Carbon Black',
sourceRef=source_ref,
artifacts=artifacts,
caseTemplate='Carbon Black Case',
customFields=customFields)
try:
ret = theHiveConnector.createAlert(alert)
logger.info('Alert {} created in TheHive'.format(
str(ret['id'])))
except ValueError:
logger.warning('Failed to create alert trying to update')
try:
ret = theHiveConnector.updateAlert(
alert.sourceRef, alert)
logger.info('Alert {} updated in TheHive'.format(
str(ret['id'])))
except ValueError as error:
logger.error(
"Failed to create alert ! {}".format(error))
result['success'] = True
except Exception as error:
result['success'] = False
result['message'] = str(error)
return result
def connectEws():
logger = logging.getLogger(__name__)
logger.info('%s.connectEws starts', __name__)
report = dict()
report['success'] = bool()
try:
cfg = getConf()
ewsConnector = EwsConnector(cfg)
folder_name = cfg.get('EWS', 'folder_name')
unread = ewsConnector.scan(folder_name)
theHiveConnector = TheHiveConnector(cfg)
for msg in reversed(unread):
#type(msg)
#<class 'exchangelib.folders.Message'>
conversationId = msg.conversation_id.id
#searching if case has already been created from the email
#conversation
esCaseId = theHiveConnector.searchCaseByDescription(conversationId)
if esCaseId is None:
#no case previously created from the conversation
caseTitle = str(msg.subject)
caseDescription = ('```\n' + 'Case created by Synapse\n' +
'conversation_id: "' +
str(msg.conversation_id.id) + '"\n' + '```')
if msg.categories:
assignee = msg.categories[0]
else:
assignee = 'synapse'
case = theHiveConnector.craftCase(caseTitle, caseDescription)
createdCase = theHiveConnector.createCase(case)
caseUpdated = theHiveConnector.assignCase(
createdCase, assignee)
commTask = theHiveConnector.craftCommTask()
esCaseId = caseUpdated.id
commTaskId = theHiveConnector.createTask(esCaseId, commTask)
else:
#case previously created from the conversation
commTaskId = theHiveConnector.getTaskIdByTitle(
esCaseId, 'Communication')
if commTaskId != None:
pass
else:
#case already exists but no Communication task found
#creating comm task
commTask = theHiveConnector.craftCommTask()
commTaskId = theHiveConnector.createTask(
esCaseId, commTask)
fullBody = getEmailBody(msg)
taskLog = theHiveConnector.craftTaskLog(fullBody)
createdTaskLogId = theHiveConnector.addTaskLog(commTaskId, taskLog)
attachedFiles = getFileAttachments(msg)
for attached in attachedFiles:
theHiveConnector.addFileObservable(esCaseId, attached['data'],
attached['message'])
readMsg = ewsConnector.markAsRead(msg)
report['success'] = True
return report
except Exception as e:
logger.error('Failed to create case from email', exc_info=True)
report['success'] = False
return report
def connectEws():
logger = logging.getLogger(__name__)
logger.info('%s.connectEws starts', __name__)
report = dict()
report['success'] = bool()
try:
cfg = getConf()
ewsConnector = EwsConnector(cfg)
folder_name = cfg.get('EWS', 'folder_name')
unread = ewsConnector.scan(folder_name)
logger.info("Found " + str(len(unread)) + " unreads mails")
theHiveConnector = TheHiveConnector(cfg)
for msg in unread:
#type(msg)
#<class 'exchangelib.folders.Message'>
conversationId = msg.conversation_id.id
#searching if case has already been created from the email
#conversation
esCaseId = theHiveConnector.searchCaseByDescription(conversationId)
if esCaseId is None:
#no case previously created from the conversation
caseTitle = str(msg.subject)
caseDescription = ('```\n' + 'Case created by Synapse\n' +
'conversation_id: "' +
str(msg.conversation_id.id) + '"\n' + '```')
if msg.categories:
assignee = msg.categories[0]
else:
assignee = 'synapse'
case = theHiveConnector.craftCase(caseTitle, caseDescription)
createdCase = theHiveConnector.createCase(case)
caseUpdated = theHiveConnector.assignCase(
createdCase, assignee)
commTask = theHiveConnector.craftCommTask()
esCaseId = caseUpdated.id
commTaskId = theHiveConnector.createTask(esCaseId, commTask)
else:
#case previously created from the conversation
commTaskId = theHiveConnector.getTaskIdByTitle(
esCaseId, 'Communication')
if commTaskId != None:
pass
else:
#case already exists but no Communication task found
#creating comm task
commTask = theHiveConnector.craftCommTask()
commTaskId = theHiveConnector.createTask(
esCaseId, commTask)
fullBody = getEmailBody(msg)
taskLog = theHiveConnector.craftTaskLog(fullBody)
createdTaskLogId = theHiveConnector.addTaskLog(commTaskId, taskLog)
serverIPs = re.findall(IP_EX, fullBody)
for ip in serverIPs:
logger.info("IP address found in email: " + ip)
try:
r_id = theHiveConnector.addIPObservable(esCaseId, ip, '')
result = theHiveConnector.scanIP(r_id)
except ValueError as ex:
logger.info(ex)
readMsg = ewsConnector.markAsRead(msg)
for attachmentLvl1 in msg.attachments:
#uploading the attachment as file observable
#is the attachment is a .msg, the eml version
#of the file is uploaded
tempAttachment = TempAttachment(attachmentLvl1)
if not tempAttachment.isInline:
#adding the attachment only if it is not inline
#inline attachments are pictures in the email body
tmpFilepath = tempAttachment.writeFile()
to = str()
for recipient in msg.to_recipients:
to = to + recipient.email_address + ' '
comment = 'Attachment from email sent by '
comment += str(msg.author.email_address).lower()
comment += ' and received by '
comment += str(to).lower()
comment += ' with subject: <'
comment += msg.subject
comment += '>'
theHiveConnector.addFileObservable(esCaseId, tmpFilepath,
comment)
if tempAttachment.isEmailAttachment:
#if the attachment is an email
#attachments of this email are also
#uploaded to TheHive
for attachmentLvl2 in tempAttachment.attachments:
tempAttachmentLvl2 = TempAttachment(attachmentLvl2)
tmpFilepath = tempAttachmentLvl2.writeFile()
comment = 'Attachment from the email attached'
theHiveConnector.addFileObservable(
esCaseId, tmpFilepath, comment)
report['success'] = True
return report
except Exception as e:
logger.error('Failed to create case from email', exc_info=True)
report['success'] = False
return report
def rapid7IDRAlerts2Alerts(alert_data, org_name):
logger = logging.getLogger('workflows.' + __name__)
logger.info('%s.rapid7IDRAlerts2Alert starts', __name__)
result = {}
result['success'] = bool()
conf = getConf()
theHiveConnector = TheHiveConnector(conf)
logger.info("Building custom fields ...")
customFields = CustomFieldHelper()\
.add_string('client', org_name)\
.build()
tags = []
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
alert_date = dateutil_parser.parse(alert_data.get('timestamp', now))
for user in alert_data.get('actors', {}).get('users', []):
tags.append(user.get('name', ""))
for asset in alert_data.get('actors', {}).get('assets', []):
tags.append(asset.get('shortname', ""))
logger.info("Building description ...")
description = descriptionCrafter(alert_data, org_name)
logger.info("Building alert ...")
alert = theHiveConnector.craftAlert(
title=alert_data.get(
'title', alert_data.get('name',
"New alert from Rapid7 Insight IDR")),
description=description,
severity=2, #is there a way to determine efficiently this thing ?
date=int(alert_date.timestamp()) * 1000,
tags=tags,
tlp=2,
status="New",
type="SIEM",
source="Rapid7 Insight IDR",
sourceRef=alert_data.get('investigationId', str(uuid.uuid4())),
artifacts=artifactsCrafter(alert_data),
caseTemplate="Insight IDR Case",
customFields=customFields)
logger.info("Sending alert to TheHive ...")
try:
ret = theHiveConnector.createAlert(alert)
logger.info("Alert {} created in TheHive".format(str(ret['id'])))
result['success'] = True
except ValueError:
logger.warning("Alert creation failed, trying to update ...")
try:
ret = theHiveConnector.updateAlert(alert.sourceRef, alert)
logger.info("Alert {} updated in TheHive".format(str(ret['id'])))
result['success'] = True
except Exception as error:
logger.error("Alert update failed ! {}".format(error))
result['success'] = False
return result
def threats2Alert():
logger = logging.getLogger('workflows.' + __name__)
logger.info('%s.threats2Alert starts', __name__)
theHiveConnector = TheHiveConnector(getConf())
organizations = None
with open(os.path.join(current_dir, "..", "conf", "sentinelone.json")) as fd:
organizations = json.load(fd)['organizations']
for org in organizations:
threats = SentinelOneConnector(org['mgt_url'], org['token']).get_threats()
for threat in threats:
tags = []
for indicator in threat['indicators']:
tags.append(indicator['category'])
for tactic in indicator['tactics']:
tags.append(tactic['name'])
customFields = CustomFieldHelper()\
.add_string('client', org['name'])\
.add_string('hostname', str(threat['agentRealtimeInfo']['agentComputerName']))\
.build()
sourceRef = "{}-{}".format(org['trigram'],threat['id'])
severity = {"na":1, "suspicious":2, "malicious":3}[threat['threatInfo']['confidenceLevel']]
# external link attribute is TheHive 4 only
alert = Alert(
title="{} performing {} activity".format(threat['threatInfo']['threatName'], threat['threatInfo']['classification']),
description=descriptionCrafter(threat, org),
severity=severity,
tags=tags,
tlp=2,
date=int(dateutil_parse(threat['threatInfo']['createdAt']).timestamp()) * 1000,
status="New",
type='EDR',
source='Sentinel One',
sourceRef=sourceRef,
artifacts=artifactsCrafter(threat),
caseTemplate='Sentinel One Case',
customFields=customFields,
externalLink="{mgt_url}/incidents/threats/{threat_id}/overview"\
.format(mgt_url=org['mgt_url'], threat_id=threat['id']))
find = theHiveConnector.theHiveApi.find_alerts(query=Eq('sourceRef', sourceRef))
find.raise_for_status()
find = find.json()
if len(find) > 0:
pass
# update because already exists
# disabled for now
#ret = theHiveConnector.theHiveApi.update_alert(find['id'], alert)
#ret.raise_for_status()
#logger.info("Alert {} updated in TheHive".format(find['id']))
else:
# create because does not exists in TheHive
ret = theHiveConnector.theHiveApi.create_alert(alert)
ret.raise_for_status()
logger.info('Alert {} created in TheHive'.format(ret.json()['id']))
return {'success': True, 'message': ""}
