def _create_alert_config(rule, match):
context = {'rule': rule, 'match': match}
alert_config = {
'artifacts': _create_artifacts(rule, match),
'sourceRef': str(uuid.uuid4())[0:6],
'title': '{rule[name]}'.format(**context)
}
alert_config.update(rule.get('hive_alert_config', {}))
for alert_config_field, alert_config_value in alert_config.items():
if alert_config_field == 'customFields':
custom_fields = CustomFieldHelper()
for cf_key, cf_value in alert_config_value.items():
try:
func = getattr(custom_fields, 'add_{}'.format(cf_value['type']))
except AttributeError:
raise Exception('unsupported custom field type {}'.format(cf_value['type']))
value = cf_value['value'].format(**context)
func(cf_key, value)
alert_config[alert_config_field] = custom_fields.build()
elif isinstance(alert_config_value, str):
alert_config[alert_config_field] = alert_config_value.format(**context)
elif isinstance(alert_config_value, (list, tuple)):
formatted_list = []
for element in alert_config_value:
try:
formatted_list.append(element.format(**context))
except (AttributeError, KeyError, IndexError):
formatted_list.append(element)
alert_config[alert_config_field] = formatted_list
return alert_config
def alert(self, matches):
connection_details = self.rule['hive_connection']
api = TheHiveApi(
'{hive_host}:{hive_port}'.format(**connection_details),
connection_details.get('hive_apikey',''),
proxies=connection_details.get('hive_proxies', {'http': '', 'https': ''}),
cert=connection_details.get('hive_verify', False))
for match in matches:
context = {'rule': self.rule, 'match': match}
artifacts = []
for mapping in self.rule.get('hive_observable_data_mapping', []):
for observable_type, match_data_key in mapping.iteritems():
try:
match_data_keys = re.findall(r'\{match\[([^\]]*)\]', match_data_key)
if all([True for k in match_data_keys if k in context['match']]):
artifacts.append(AlertArtifact(dataType=observable_type, data=match_data_key.format(**context)))
except KeyError:
raise KeyError('\nformat string\n{}\nmatch data\n{}'.format(match_data_key, context))
alert_config = {
'artifacts': artifacts,
'sourceRef': str(uuid.uuid4())[0:6],
'title': '{rule[name]}'.format(**context)
}
alert_config.update(self.rule.get('hive_alert_config', {}))
for alert_config_field, alert_config_value in alert_config.iteritems():
if alert_config_field == 'customFields':
custom_fields = CustomFieldHelper()
for cf_key, cf_value in alert_config_value.iteritems():
try:
func = getattr(custom_fields, 'add_{}'.format(cf_value['type']))
except AttributeError:
raise Exception('unsupported custom field type {}'.format(cf_value['type']))
value = cf_value['value'].format(**context)
func(cf_key, value)
alert_config[alert_config_field] = custom_fields.build()
elif isinstance(alert_config_value, basestring):
alert_config[alert_config_field] = alert_config_value.format(**context)
elif isinstance(alert_config_value, (list, tuple)):
formatted_list = []
for element in alert_config_value:
try:
formatted_list.append(element.format(**context))
except:
formatted_list.append(element)
alert_config[alert_config_field] = formatted_list
alert = Alert(**alert_config)
response = api.create_alert(alert)
if response.status_code != 201:
raise Exception('alert not successfully created in TheHive\n{}'.format(response.text))
def create_alarm(self,
title,
source_ref=None,
description='N/A',
alert_type='external',
source='LogRhythm',
iocs=None,
additional_fields=None,
additional_tags=None,
tlp=TLP.AMBER,
pap=PAP.AMBER,
severity=HiveSeverity.MEDIUM):
if source_ref is None:
source_ref = str(uuid.uuid4())[0:6]
alert_tags = self.alert_tags.copy()
if additional_tags is not None:
for additional_tag in additional_tags:
alert_tags.append(additional_tag)
custom_fields_helper = CustomFieldHelper()
if additional_fields is not None:
for field in additional_fields:
custom_fields_helper.add_string(field['name'], field['value'])
custom_fields = custom_fields_helper.build()
artifacts = list()
if iocs is not None:
for ioc in iocs:
artifacts.append(
AlertArtifact(dataType=ioc['type'].value,
data=ioc['value']))
hive_alert = Alert(title=title,
tlp=tlp.value,
tags=alert_tags,
description=description,
type=alert_type,
source=source,
sourceRef=source_ref,
pap=pap.value,
artifacts=artifacts,
customFields=custom_fields,
severity=severity.value)
response = self.api.create_alert(hive_alert)
if response.status_code == 201:
print('Alerta Creada Exitosamente')
print(json.dumps(response.json(), indent=4, sort_keys=True))
else:
print('Error')
print(response.text)
return response.json()
def create_case_from_inc_name(api, incident):
# Prepare the custom fields
customFields = CustomFieldHelper() \
.add_string('category', incident["category"]) \
.add_string('type', incident["type"]) \
.build()
# Словарь соответствия Severity в MP SIEM и The HIVE
severity_dict = {'Low': 1, 'Medium': 2, 'High': 3}
case = Case(title=incident["name"],
tlp=3,
flag=True,
tags=['MP SIEM', 'Script'],
description='',
severity=severity_dict[incident["severity"]],
customFields=customFields)
# Create the CASE
response = api.create_case(case)
if response.status_code == 201:
# print(json.dumps(response.json(), indent=4, sort_keys=True))
case_id = response.json()['id']
print_log("INFO: New Case **" + incident["name"] +
"** in The Hive created")
else:
print_log('ERROR: {}/{}'.format(response.status_code, response.text))
sys.exit(0)
return case_id
def CreateHiveAlertFromSentinel(api, title, description, incidentnumber,
severity, source, artifacts, alertIds,
incidentURL):
tags = []
if severity == "Low":
theHiveSeverity = 1
elif severity == "Informational":
theHiveSeverity = 1
elif severity == "Medium":
theHiveSeverity = 2
elif severity == "High":
theHiveSeverity = 3
alertIdsStr = ' '.join(map(str, alertIds))
customFields = CustomFieldHelper()
customFields.add_number('sentinelIncidentNumber', incidentnumber)
customFields.add_string('alertIds', alertIdsStr)
customFields.add_string('incidentURL', incidentURL)
customFields = customFields.build()
alert = Alert(title=title,
tlp=2,
tags=tags,
description=description,
type='Sentinel',
severity=theHiveSeverity,
source='Sentinel:' + source,
customFields=customFields,
sourceRef="Sentinel" + str(incidentnumber),
artifacts=artifacts)
# Create the Alert
response = api.create_alert(alert)
if response.status_code == 201:
logging.info(
'Alert created: ' + 'Sentinel' + str(incidentnumber) + ': ' +
source + ': ' + title +
'. StatusCode: {}/{}'.format(response.status_code, response.text))
elif (response.status_code == 400
and response.json()['type'] == "ConflictError"):
logging.info(
'Duplicate alert: ' + 'Sentinel' + str(incidentnumber) + ': ' +
source + ': ' + title +
'. StatusCode: {}/{}'.format(response.status_code, response.text))
else:
logging.error(
'failed to create alert: ' + source + ' ' + title + ' Sentinel' +
str(incidentnumber) +
'. StatusCode: {}/{}'.format(response.status_code, response.text))
sys.exit(0)
def create_case(self,
title,
tasks=None,
tlp=TLP.AMBER,
pap=PAP.AMBER,
severity=HiveSeverity.MEDIUM,
additional_fields=None,
additional_tags=None,
flag=False,
description='N/A'):
case_tags = self.case_tags.copy()
if additional_tags is not None:
for additional_tag in additional_tags:
case_tags.append(additional_tag)
custom_fields_helper = CustomFieldHelper()
if additional_fields is not None:
for field in additional_fields:
custom_fields_helper.add_string(field['name'], field['value'])
custom_fields = custom_fields_helper.build()
new_tasks = list()
if tasks is not None:
for task in tasks:
new_tasks.append(CaseTask(title=task))
hive_case = Case(title=title,
tlp=tlp.value,
pap=pap.value,
description=description,
tags=case_tags,
severity=severity.value,
flag=flag,
customFields=custom_fields,
tasks=new_tasks)
response = self.api.create_case(hive_case)
if response.status_code == 201:
print('Caso Creada Exitosamente')
print(json.dumps(response.json(), indent=4, sort_keys=True))
else:
print('Error')
print(response.text)
return response.json()
def test_intger():
custom_fields = CustomFieldHelper() \
.add_float('float-field', 1.0) \
.build()
assert custom_fields is not None
assert 'float-field' in custom_fields
assert 'float' in custom_fields['float-field']
assert 'order' in custom_fields['float-field']
assert custom_fields['float-field']['float'] == 1.0
assert custom_fields['float-field']['order'] == 0
def test_intger():
custom_fields = CustomFieldHelper() \
.add_integer('integer-field', 100) \
.build()
assert custom_fields is not None
assert 'integer-field' in custom_fields
assert 'integer' in custom_fields['integer-field']
assert 'order' in custom_fields['integer-field']
assert custom_fields['integer-field']['integer'] == 100
assert custom_fields['integer-field']['order'] == 0
def test_boolean():
custom_fields = CustomFieldHelper() \
.add_boolean('boolean-field', True) \
.build()
assert custom_fields is not None
assert 'boolean-field' in custom_fields
assert 'boolean' in custom_fields['boolean-field']
assert 'order' in custom_fields['boolean-field']
assert custom_fields['boolean-field']['boolean'] is True
assert custom_fields['boolean-field']['order'] == 0
def test_string():
custom_fields = CustomFieldHelper() \
.add_string('string-field', 'thehive') \
.build()
assert custom_fields is not None
assert 'string-field' in custom_fields
assert 'string' in custom_fields['string-field']
assert 'order' in custom_fields['string-field']
assert custom_fields['string-field']['string'] == 'thehive'
assert custom_fields['string-field']['order'] == 0
def test_date():
date = int(time.time()) * 1000
custom_fields = CustomFieldHelper() \
.add_date('date-field', date) \
.build()
assert custom_fields is not None
assert 'date-field' in custom_fields
assert 'date' in custom_fields['date-field']
assert 'order' in custom_fields['date-field']
assert custom_fields['date-field']['date'] == date
assert custom_fields['date-field']['order'] == 0
def __init__(self, event, tenant):
# Case attributes
user = event['user']
nature = event['nature']
case_name = event['case']['name']
case_id = event['case']['id']
case_url = event['case']['url']
case_tags = ['McAfee Investigator', case_id, nature]
# Task attributes
task_owner = tenant.hive_user
# task_start_date = datetime.datetime.now().strftime("%d-%m-%Y %H:%M")
task_start_date = int(time.time()) * 1000
task_description = "Autogenerated task to track MI case events"
tasks = [
CaseTask(title='MI - analytics investigation',
status='InProgress',
owner=task_owner,
startDate=task_start_date,
description=task_description),
CaseTask(title='MI - system events',
status='InProgress',
owner=task_owner,
startDate=task_start_date,
description=task_description),
CaseTask(title='MI - user events',
status='Waiting',
description="Auto generated task to track MI user events")
]
# Prepare the custom fields (values are searchable)
customFields = CustomFieldHelper()\
.add_boolean('MI-autogenerated', True)\
.add_string('MI-caseid', case_id)\
.add_string('MI-user', user)\
.add_string('MI-nature', nature)\
.build()
case = Case(title=case_name,
tlp=0,
flag=False,
description="For additional info see " +
self.get_case_ui_url(case_url),
tags=case_tags,
tasks=tasks,
customFields=customFields)
self.case = case
def sendtoHIVE(title, description, domain):
tasks = [
CaseTask(title='Tracking'),
CaseTask(title='Communication'),
CaseTask(title='Investigation', status='Waiting', flag=True)
]
# Prepare the custom fields
customFields = CustomFieldHelper()\
.add_boolean('booleanField', True)\
.add_string('businessImpact', 'HIGH')\
.add_date('occurDate', int(time.time())*1000)\
.add_number('cvss', 9)\
.build()
case = Case(title=title,
tlp=3,
flag=True,
tags=['wordpress', domain],
description=description,
tasks=tasks,
customFields=customFields)
# Create the case
print('Create Case')
print('-----------------------------')
id = None
response = api.create_case(case)
if response.status_code == 201:
print(json.dumps(response.json(), indent=4, sort_keys=True))
print('')
id = response.json()['id']
else:
print('ko: {}/{}'.format(response.status_code, response.text))
sys.exit(0)
# Get all the details of the created case
print('Get created case {}'.format(id))
print('-----------------------------')
response = api.get_case(id)
if response.status_code == requests.codes.ok:
print(json.dumps(response.json(), indent=4, sort_keys=True))
print('')
else:
print('ko: {}/{}'.format(response.status_code, response.text))
def parse_hooks(self):
logger.info('Start parsing webhooks for ELK automation')
#Set the default action
self.report_action = 'None'
#if it is a new case
if self.webhook.isImportedAlert():
logger.info('Registered the finishing steps of the creation of a new case')
#Register case info
self.caseid = self.webhook.data['details']['case']
self.tags = self.webhook.data['object']['tags']
self.description = self.webhook.data['object']['description']
#Enrich the case information with missing information from the alert
if 'ELK' in self.webhook.data['object']['tags']:
self.report_action = 'enrichCase'
logger.info('Alert {} has been tagged as ELK and is just imported or following has been reenabled. Adding custom fields'.format(self.webhook.data['details']['case']))
# Enrich offense with information from the alert by posting the missing information through an API call
#Create a Case object
self.case = Case()
#Add the case id to the object
self.case.id = self.caseid
logger.info('Updating case %s' % self.case.id)
#Define which fields need to get updated
fields = ['customFields']
#Retrieve all required attributes from the alert and add them as custom fields to the case
customFields = CustomFieldHelper()\
.add_string('anomalyType', self.webhook.data['object']['type'])\
.add_string('source', self.webhook.data['object']['source'])\
.build()
#Add custom fields to the case object
self.case.customFields = customFields
#Update the case
response = self.TheHiveConnector.updateCase(self.case,fields)
return report_action
def test_order():
custom_fields = CustomFieldHelper()\
.add_boolean('booleanField', True)\
.add_string('businessImpact', 'HIGH')\
.add_date('occurDate', int(time.time())*1000)\
.add_number('cvss', 9) \
.add_integer('csirts', 2) \
.add_float('hitRate', 50.2) \
.build()
assert custom_fields is not None
assert 'booleanField' in custom_fields
assert 'businessImpact' in custom_fields
assert 'occurDate' in custom_fields
assert 'cvss' in custom_fields
assert 'csirts' in custom_fields
assert 'hitRate' in custom_fields
assert custom_fields['booleanField']['order'] == 0
assert custom_fields['businessImpact']['order'] == 1
assert custom_fields['occurDate']['order'] == 2
assert custom_fields['cvss']['order'] == 3
assert custom_fields['csirts']['order'] == 4
assert custom_fields['hitRate']['order'] == 5
def rapid7IDRAlerts2Alerts(alert_data, org_name):
logger = logging.getLogger('workflows.' + __name__)
logger.info('%s.rapid7IDRAlerts2Alert starts', __name__)
result = {}
result['success'] = bool()
conf = getConf()
theHiveConnector = TheHiveConnector(conf)
logger.info("Building custom fields ...")
customFields = CustomFieldHelper()\
.add_string('client', org_name)\
.build()
tags = []
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
alert_date = dateutil_parser.parse(alert_data.get('timestamp', now))
for user in alert_data.get('actors', {}).get('users', []):
tags.append(user.get('name', ""))
for asset in alert_data.get('actors', {}).get('assets', []):
tags.append(asset.get('shortname', ""))
logger.info("Building description ...")
description = descriptionCrafter(alert_data, org_name)
logger.info("Building alert ...")
alert = theHiveConnector.craftAlert(
title=alert_data.get(
'title', alert_data.get('name',
"New alert from Rapid7 Insight IDR")),
description=description,
severity=2, #is there a way to determine efficiently this thing ?
date=int(alert_date.timestamp()) * 1000,
tags=tags,
tlp=2,
status="New",
type="SIEM",
source="Rapid7 Insight IDR",
sourceRef=alert_data.get('investigationId', str(uuid.uuid4())),
artifacts=artifactsCrafter(alert_data),
caseTemplate="Insight IDR Case",
customFields=customFields)
logger.info("Sending alert to TheHive ...")
try:
ret = theHiveConnector.createAlert(alert)
logger.info("Alert {} created in TheHive".format(str(ret['id'])))
result['success'] = True
except ValueError:
logger.warning("Alert creation failed, trying to update ...")
try:
ret = theHiveConnector.updateAlert(alert.sourceRef, alert)
logger.info("Alert {} updated in TheHive".format(str(ret['id'])))
result['success'] = True
except Exception as error:
logger.error("Alert update failed ! {}".format(error))
result['success'] = False
return result
def submitTheHive(message):
'''Create a new case in TheHive based on the email'''
# Decode email
msg = email.message_from_bytes(message)
decode = email.header.decode_header(msg['From'])[0]
fromField = str(decode[0])
decode = email.header.decode_header(msg['Subject'])[0]
subjectField = str(decode[0])
if args.verbose:
print("[INFO] From: %s Subject: %s" % (fromField, subjectField))
attachments = []
body = ''
for part in msg.walk():
if part.get_content_type() == "text/plain":
body = part.get_payload(decode=True).decode()
else:
# Extract MIME parts
filename = part.get_filename()
mimetype = part.get_content_type()
if filename and mimetype:
if mimetype in config['caseFiles'] or not config['caseFiles']:
print("[INFO] Found attachment: %s (%s)" %
(filename, mimetype))
# Decode the attachment and save it in a temporary file
charset = part.get_content_charset()
if charset is None:
charset = chardet.detect(bytes(part))['encoding']
fd, path = tempfile.mkstemp(prefix=slugify(filename) + "_")
try:
with os.fdopen(fd, 'w+b') as tmp:
tmp.write(part.get_payload(decode=1))
attachments.append(path)
except OSError as e:
print("[ERROR] Cannot dump attachment to %s: %s" %
(path, e.errno))
api = TheHiveApi(config['thehiveURL'], config['thehiveUser'],
config['thehivePassword'], {
'http': '',
'https': ''
})
if '[ALERT]' in subjectField:
# Prepare the alert
sourceRef = str(uuid.uuid4())[0:6]
alert = Alert(title=subjectField.replace('[ALERT]', ''),
tlp=int(config['alertTLP']),
tags=config['alertTags'],
description=body,
type='external',
source=fromField,
sourceRef=sourceRef)
# Create the Alert
id = None
response = api.create_alert(alert)
if response.status_code == 201:
if args.verbose:
print('[INFO] Created alert %s' % response.json()['sourceRef'])
else:
print('[ERROR] Cannot create alert: %s (%s)' %
(response.status_code, response.text))
sys.exit(0)
else:
# Prepare the sample case
tasks = []
for task in config['caseTasks']:
tasks.append(CaseTask(title=task))
# Prepare the custom fields
customFields = CustomFieldHelper()\
.add_string('from', fromField)\
.add_string('attachment', str(attachments))\
.build()
case = Case(title=subjectField,
tlp=int(config['caseTLP']),
flag=False,
tags=config['caseTags'],
description=body,
tasks=tasks,
customFields=customFields)
# Create the case
id = None
response = api.create_case(case)
if response.status_code == 201:
newID = response.json()['id']
if args.verbose:
print('[INFO] Created case %s' % response.json()['caseId'])
if len(attachments) > 0:
for path in attachments:
observable = CaseObservable(
dataType='file',
data=[path],
tlp=int(config['caseTLP']),
ioc=False,
tags=config['caseTags'],
message='Created by imap2thehive.py')
response = api.create_case_observable(newID, observable)
if response.status_code == 201:
if args.verbose:
print('[INFO] Added observable %s to case ID %s' %
(path, newID))
os.unlink(path)
else:
print('[ERROR] Cannot add observable: %s - %s (%s)' %
(path, response.status_code, response.text))
sys.exit(0)
else:
print('[ERROR] Cannot create case: %s (%s)' %
(response.status_code, response.text))
sys.exit(0)
return
def submitTheHive(message):
'''
Create a new case in TheHive based on the email
Return 'TRUE' is successfully processed otherwise 'FALSE'
'''
global log
# Decode email
msg = email.message_from_bytes(message)
decode = email.header.decode_header(msg['From'])[0]
if decode[1] is not None:
fromField = decode[0].decode(decode[1])
else:
fromField = str(decode[0])
decode = email.header.decode_header(msg['Subject'])[0]
if decode[1] is not None:
subjectField = decode[0].decode(decode[1])
else:
subjectField = str(decode[0])
log.info("From: %s Subject: %s" % (fromField, subjectField))
attachments = []
observables = []
# Extract SMTP headers and search for observables
parser = HeaderParser()
headers = parser.parsestr(msg.as_string())
headers_string = ''
i = 0
while i < len(headers.keys()):
headers_string = headers_string + headers.keys(
)[i] + ': ' + headers.values()[i] + '\n'
i += 1
# Temporary disabled
# observables = searchObservables(headers_string, observables)
body = ''
for part in msg.walk():
if part.get_content_type() == "text/plain":
try:
body = part.get_payload(decode=True).decode()
except UnicodeDecodeError:
body = part.get_payload(decode=True).decode('ISO-8859-1')
observables.extend(searchObservables(body, observables))
elif part.get_content_type() == "text/html":
try:
html = part.get_payload(decode=True).decode()
except UnicodeDecodeError:
html = part.get_payload(decode=True).decode('ISO-8859-1')
observables.extend(searchObservables(html, observables))
else:
# Extract MIME parts
filename = part.get_filename()
mimetype = part.get_content_type()
if filename and mimetype:
if mimetype in config['caseFiles'] or not config['caseFiles']:
log.info("Found attachment: %s (%s)" %
(filename, mimetype))
# Decode the attachment and save it in a temporary file
charset = part.get_content_charset()
if charset is None:
charset = chardet.detect(bytes(part))['encoding']
fd, path = tempfile.mkstemp(prefix=slugify(filename) + "_")
try:
with os.fdopen(fd, 'w+b') as tmp:
tmp.write(part.get_payload(decode=1))
attachments.append(path)
except OSerror as e:
log.error("Cannot dump attachment to %s: %s" %
(path, e.errno))
return False
# Cleanup observables (remove duplicates)
new_observables = []
for o in observables:
if not {'type': o['type'], 'value': o['value']} in new_observables:
# Is the observable whitelisted?
if isWhitelisted(o['value']):
log.debug('Skipping whitelisted observable: %s' % o['value'])
else:
new_observables.append({
'type': o['type'],
'value': o['value']
})
log.debug('Found observable %s: %s' % (o['type'], o['value']))
else:
log.info('Ignoring duplicate observable: %s' % o['value'])
log.info("Removed duplicate observables: %d -> %d" %
(len(observables), len(new_observables)))
observables = new_observables
if config['thehiveAuthMethod'] == 'APIKey':
api = TheHiveApi(config['thehiveURL'], config['thehiveAPIKey'], None, {
'http': '',
'https': ''
})
else:
api = TheHiveApi(config['thehiveURL'], config['thehiveUser'],
config['thehivePassword'], {
'http': '',
'https': ''
})
# Search for interesting keywords in subjectField:
log.debug("Searching for %s in '%s'" %
(config['alertKeywords'], subjectField))
if re.match(config['alertKeywords'], subjectField, flags=0):
#
# Add observables found in the mail body
#
artifacts = []
if config['thehiveObservables'] and len(observables) > 0:
for o in observables:
artifacts.append(
AlertArtifact(dataType=o['type'], data=o['value']))
#
# Prepare tags - add alert keywords found to the list of tags
#
tags = list(config['alertTags'])
match = re.findall(config['alertKeywords'], subjectField)
for m in match:
tags.append(m)
#
# Prepare the alert
#
sourceRef = str(uuid.uuid4())[0:6]
alert = Alert(title=subjectField.replace('[ALERT]', ''),
tlp=int(config['alertTLP']),
tags=tags,
description=body,
type='external',
source=fromField,
sourceRef=sourceRef,
artifacts=artifacts)
# Create the Alert
id = None
response = api.create_alert(alert)
if response.status_code == 201:
log.info('Created alert %s' % response.json()['sourceRef'])
else:
log.error('Cannot create alert: %s (%s)' %
(response.status_code, response.text))
return False
else:
# Prepare the sample case
tasks = []
for task in config['caseTasks']:
tasks.append(CaseTask(title=task))
# Prepare the custom fields
customFields = CustomFieldHelper()\
.add_string('from', fromField)\
.add_string('attachment', str(attachments))\
.build()
# If a case template is specified, use it instead of the tasks
if len(config['caseTemplate']) > 0:
case = Case(title=subjectField,
tlp=int(config['caseTLP']),
flag=False,
tags=config['caseTags'],
description=body,
template=config['caseTemplate'],
customFields=customFields)
else:
def connectEws():
report = dict()
report['success'] = bool()
clients = None
with open('/opt/Synapse/conf/mailboxes.json') as f:
clients = json.load(f)
mailboxes = clients['mailboxes']
cfg = getConf()
theHiveConnector = TheHiveConnector(cfg)
for mailbox in mailboxes:
ewsConnector = EwsConnector(username=mailbox['username'],
password=mailbox['password'],
auth_type=mailbox['auth_type'],
server=mailbox['server'],
smtp_address=mailbox['smtp_address'])
unread = ewsConnector.scan(mailbox['folder_name'])
for msg in unread:
#type(msg)
#<class 'exchangelib.folders.Message'>
conversationId = msg.conversation_id.id
origin_mail = msg.attachments[0].item
fullBody = getEmailBody(msg)
alertTitle = "{}; {}".format(
msg.subject, ''.join(
[i if ord(i) < 128 else '' for i in origin_mail.subject]))
alertDescription = ('```\n' + 'Alert created by Synapse\n' +
'conversation_id: "' +
str(msg.conversation_id.id) + '"\n\n' +
'Original mail:\n\n' + fullBody[3:])
# rapporteur
reporter = str(msg.sender.email_address).lower()
# expéditeur
sender = str(origin_mail.sender.email_address).lower()
# date de réception originale et en-tant que spam
rcv_date_timestamp = origin_mail.datetime_received.timestamp(
) * 1000
# date de signalement à la boîte support
rpt_date_timestamp = msg.datetime_received.timestamp() * 1000
customFields = CustomFieldHelper()\
.add_string('reporter', reporter)\
.add_string('sender', sender)\
.add_date('receivedDate', rcv_date_timestamp)\
.add_date('reportedDate', rpt_date_timestamp)\
.add_string('client', mailbox['name'])\
.build()
file_paths = []
artifacts = []
artifacts.append(
AlertArtifact(dataType='mail',
data=reporter,
ignoreSimilarity=True,
tags=['Synapse']))
artifacts.append(
AlertArtifact(dataType='mail',
data=sender,
message="Original sender of the e-mail",
tags=['Synapse', 'Sender']))
artifacts.append(
AlertArtifact(dataType='mail_subject',
data=str(origin_mail.subject),
message="Original subject of the e-mail",
tags=['Synapse']))
attachedFiles = getFileAttachments(msg)
for attached in attachedFiles:
artifacts.append(
AlertArtifact(dataType='file',
data=[attached['path']],
tags=['Attachment', 'Synapse']))
file_paths.append(attached['path'])
alert = Alert(title=alertTitle,
description=alertDescription,
severity=2,
date=rcv_date_timestamp,
tags=[],
tlp=2,
status="New",
type="Phishing",
source="BAL Phishing",
sourceRef="{}-{}".format(
mailbox['short_name'],
str(uuid.uuid4())[24:].upper()),
artifacts=artifacts,
caseTemplate="Suspicious Email Case",
customFields=customFields)
theHiveConnector.createAlert(alert)
ewsConnector.markAsRead(msg)
logger.info("Cleaning temp files ...")
for temp_file in file_paths:
try:
os.remove(temp_file)
except OSError as errRm:
logger.error("'{}' could not be removed.{}".format(
temp_file, errRm))
report['success'] = True
return report
def allOffense2Alert(timerange):
"""
Get all openned offense created within the last
<timerange> minutes and creates alerts for them in
TheHive
"""
logger = logging.getLogger(__name__)
logger.info('%s.allOffense2Alert starts', __name__)
report = dict()
report['success'] = True
report['offenses'] = list()
try:
cfg = getConf()
qradarConnector = QRadarConnector(cfg)
theHiveConnector = TheHiveConnector(cfg)
offensesList = qradarConnector.getOffenses(timerange)
#each offenses in the list is represented as a dict
#we enrich this dict with additional details
for offense in offensesList:
#searching if the offense has already been converted to alert
logger.info('Looking for offense %s in TheHive alerts',
str(offense['id']))
# Update only new Alerts, as Ignored it will be closed on QRadar and should not be updated,
# as Imported we will do a responder to fetch latest info in the case
results = theHiveConnector.findAlert(
Eq("sourceRef", str(offense['id'])))
offense_report = dict()
try:
if len(results) == 0:
logger.info(
'Offense %s not found in TheHive alerts, creating it',
str(offense['id']))
enrichedOffense = enrichOffense(qradarConnector, offense)
theHiveAlert = qradarOffenseToHiveAlert(
theHiveConnector, enrichedOffense)
theHiveEsAlertId = theHiveConnector.createAlert(
theHiveAlert)['id']
offense_report['type'] = "Creation"
offense_report['raised_alert_id'] = theHiveEsAlertId
offense_report['qradar_offense_id'] = offense['id']
offense_report['success'] = True
report['offenses'].append(offense_report)
elif results[0]['status'] not in ['Ignored', 'Imported']:
# update alert if alert is not imported and not dimissed
# will only update 'lastEventCount' and 'lastUpdatedTime' custom fields
logger.info('Updating offense %s', str(offense['id']))
alert = Alert(json=results[0])
cf = CustomFieldHelper()
alert.title = offense['description']
if 'lastEventCount' not in alert.customFields:
alert.customFields['lastEventCount'] = {}
if 'lastUpdated' not in alert.customFields:
alert.customFields['lastUpdated'] = {}
if 'offenseSource' not in alert.customFields:
alert.customFields['offenseSource'] = {}
alert.customFields['lastEventCount']['number'] = offense[
'event_count']
alert.customFields['lastUpdated']['date'] = offense[
'last_updated_time']
alert.customFields['offenseSource']['string'] = offense[
'offense_source'] # updated maybe ?
# should improve TheHiveConnector.updateAlert() rather than using this
updatedAlert = theHiveConnector.theHiveApi.update_alert(
results[0]['id'],
alert,
fields=['customFields', 'title'])
if not updatedAlert.ok:
raise ValueError(json.dumps(updatedAlert.json()))
offense_report['type'] = "Update"
offense_report['updated_alert_id'] = updatedAlert.json(
)['id']
offense_report['qradar_offense_id'] = offense['id']
offense_report['success'] = True
report['offenses'].append(offense_report)
else:
logger.info("Offense already exists")
except Exception as e:
logger.error('%s.allOffense2Alert failed',
__name__,
exc_info=True)
offense_report['success'] = False
if isinstance(e, ValueError):
errorMessage = json.loads(str(e))['message']
offense_report['message'] = errorMessage
else:
offense_report['message'] = str(
e) + ": Couldn't raise alert in TheHive"
offense_report['offense_id'] = offense['id']
# Set overall success if any fails
report['success'] = False
except Exception as e:
logger.error(
'Failed to create alert from QRadar offense (retrieving offenses failed)',
exc_info=True)
report['success'] = False
report['message'] = "%s: Failed to create alert from offense" % str(e)
return report
sl.insert_record(
enrichment_table, 'id, enrichment_id, enrichment_type, status',
'"{}","{}","{}","{}"'.format(offense_id, aql_id, 'username',
'Open'))
records = qr.get_aql_results(aql_id)
if records:
retval = parse_get_aql(records, 'username')
if retval:
for item in retval:
if item != None:
username_list.append(item)
sl.update_record(enrichment_table, 'status',
'Closed', 'enrichment_id', aql_id)
# # Adding custom_fields values to the case - static mapping.
custom_fields = CustomFieldHelper()
custom_fields.add_number('qradar_id', offense_id)
custom_fields.add_string('offense_source', offense_source)
custom_fields.build()
tlp = offense_severity_mapper(offense_magnitude)['sev']
# #Case - Offense summary md.
build_desc = """|Offense Summary:|\n|---|\n|Offense Description: {}|\n|Source NW: {}|\n|Destination NW: {}|\n|Source IPs: {}|\n|Local Destination IPs: {}|\n|Remote Destination IPs: {}|\n|Usernames: {}|\n---\nLink to the Offense: {}""".format(
offense_desc, offense_src_nw, offense_dst_nw, source_address_list,
local_dest_list, remote_dest_list, username_list, offense_link)
# #Some sample tasks-response actions for posting in the case. Customize per your reqs.
# #TODO: You can also utilize - thehive-playbook-creator - for dynamic task/playbook assignment using your QRadar Rule groups.
tasks = [
CaseTask(title='PB:- Phase:Identification'),
api = TheHiveApi('http://127.0.0.1:9000', '**YOUR_API_KEY**')
# Prepare the sample case
tasks = [
CaseTask(title='Identify'),
CaseTask(title='Contain'),
CaseTask(title='Investigate'),
CaseTask(title='Eradicate'),
CaseTask(title='Recovery'),
CaseTask(title='Lessons Learned', status='Waiting', flag=True)
]
# Prepare the custom fields
customFields = CustomFieldHelper()\
.add_boolean('booleanField', True)\
.add_string('businessImpact', 'HIGH')\
.add_date('occurDate', int(time.time())*1000)\
.add_number('cvss', 9)\
.build()
case = Case(title='Account Compromise',
tlp=3,
flag=True,
tags=['TheHive4Py', 'sample'],
description='N/A',
tasks=tasks,
customFields=customFields)
# Create the case
print('Create Case')
print('-----------------------------')
id = None
def updateACase(caseId, bigGroup, fromField, attachments):
api = TheHiveApi(config['thehiveURL'], config['thehiveUser'],
config['thehivePassword'], {
'http': '',
'https': ''
})
resolved = parseBody("Resolved", bigGroup)
resolved = resolved.lower()
Tags = parseBody("Tags", bigGroup)
Title = parseBody("Title", bigGroup)
TLP = parseBody("TLP", bigGroup)
Description = parseBody("Description", bigGroup)
AlbertID = parseBody("Albert Id", bigGroup)
Severity = parseBody("Severity", bigGroup)
ReplaceTags = parseBody("ReplaceTags", bigGroup)
ReplaceTags = ReplaceTags.lower()
resolvedStatus = parseBody(
"Resolution Status", bigGroup
) # (Indeterminate,FalsePositive, TruePositive, Other or Duplicated)
ImpactStatus = parseBody(
"Impact Status", bigGroup) #(NoImpact, WithImpact or NotApplicable) -
Summary = parseBody("Summary", bigGroup)
query = Eq('caseId', caseId)
try:
d = api.find_first(query=query)
for key, value in d.items():
if key == "_routing":
requestCase = value
except:
print(
"unable to find the caseId in the update email, this is required to uniquely identify a case to update. ending.."
)
sys.exit(0)
try:
updated_case = api.case.update(requestCase)
print(updated_case.jsonify())
updated_case.title = Title
updated_case.description = Description
try:
TLP = int(TLP)
updated_case.tlp = TLP
except:
print("Issue with updating TLP, skipping")
try:
Severity = int(Severity)
updated_case.severity = Severity
except:
print("Issue with updating severity, skipping")
print("TAGS HERE")
print(Tags)
if len(Tags) > 1:
newTags = Tags.split(",")
if "no" in ReplaceTags: #if replaceTags is no, don't replace tags, instead, append the unique ones
for x in newTags:
if x not in updated_case.tags:
if x + "\r" in updated_case.tags:
pass
else:
updated_case.tags.append(x) #append the tag
else: #if replacetags is yes or anything else, replace all tags with specified
print("replacing tags")
updated_case.tags = newTags
# (Indeterminate,FalsePositive, TruePositive, Other or Duplicated)
correctResolutions = [
"Indeterminate", "FalsePositive", "TruePositive", "Other",
"Duplicated"
]
if resolvedStatus not in correctResolutions:
print("invalid resolution status")
pass
if resolvedStatus in correctResolutions:
updated_case.resolutionStatus = resolvedStatus
correctImpactStatus = ["NoImpact", "WithImpact", "NotApplicable"]
if ImpactStatus not in correctImpactStatus:
print("invalid impact status")
pass
if ImpactStatus in correctImpactStatus:
updated_case.impactStatus = ImpactStatus
updated_case.summary = Summary
#overwrite albert id
if len(AlbertID) > 1:
customFields = CustomFieldHelper() \
.add_string('from', fromField) \
.add_string('attachment', str(attachments)) \
.add_string('albertId', AlbertID) \
.build()
updated_case.customFields = customFields
if "yes" in resolved:
print("resolved")
resolveUpdate = api.case.update(requestCase, status='Resolved')
if "no" in resolved:
print("OPEN")
resolveUpdate = api.case.update(requestCase, status='Open')
api.update_case(updated_case)
except FileExistsError as e:
print("Error updating case. {}".format(e))
def submitTheHive(message):
'''
Create a new case in TheHive based on the email
Return 'TRUE' is successfully processed otherwise 'FALSE'
'''
# Decode email
msg = email.message_from_bytes(message) # gets full content of the email
decode = email.header.decode_header(msg['From'])[0]
fromField = str(decode[0])
decode = email.header.decode_header(msg['Subject'])[0]
subjectField = str(decode[0])
if args.verbose:
print("[INFO] From: %s Subject: %s" % (fromField, subjectField))
attachments = []
observables = []
body = ''
bodyMessage = ''
for part in msg.walk():
if part.get_content_type() == "text/plain":
body = part.get_payload(decode=True).decode()
bodyMessage += body
observables = searchObservables(
body, observables
) # searches the body of the email for supplied observables
elif part.get_content_type(
) == "text/html": # if email is html based will search throuh html source code
if args.verbose:
print("[INFO] Searching for observable in HTML code")
html = part.get_payload(decode=True).decode()
observables = searchObservables(html, observables)
elif part.get_content_type(
) == "application/vnd.ms-excel": #ONLY WORKS FOR .CSV
body = part.get_payload(decode=True).decode('UTF-8')
observables = searchObservables(body, observables)
else:
# Extract MIME parts
filename = part.get_filename()
mimetype = part.get_content_type()
if filename and mimetype:
if mimetype in config['caseFiles'] or not config['caseFiles']:
print("[INFO] Found attachment: %s (%s)" %
(filename, mimetype))
# Decode the attachment and save it in a temporary file
charset = part.get_content_charset()
if charset is None:
charset = chardet.detect(bytes(part))['encoding']
fd, path = tempfile.mkstemp(prefix=slugify(filename) + "_")
try:
with os.fdopen(fd, 'w+b') as tmp:
tmp.write(part.get_payload(decode=1))
attachments.append(path)
except OSError as e:
print("[ERROR] Cannot dump attachment to %s: %s" %
(path, e.errno))
return False
api = TheHiveApi(config['thehiveURL'], config['thehiveUser'],
config['thehivePassword'], {
'http': '',
'https': ''
})
# if '[ALERT]' in subjectField:
if re.match(config['alertKeywords'], subjectField, flags=0):
#
# Add observables found in the mail body
#
artifacts = []
if config['thehiveObservables'] and len(observables) > 0:
print("t1")
for o in observables:
print("t2")
artifacts.append(
AlertArtifact(dataType=o['type'], data=o['value']))
#
# Prepare tags - add alert keywords found to the list of tags
#
tags = config['alertTags']
match = re.findall(config['alertKeywords'], subjectField)
for m in match:
tags.append(m)
#
# Prepare the alert
#
sourceRef = str(uuid.uuid4())[0:6]
alert = Alert(
title=subjectField.replace('[ALERT]', ''),
tlp=int(
config['alertTLP']
), #setting it blank since custom template allows default color, set it back to tlp = int for conf value
tags=tags,
description=body,
type='external',
source=fromField,
sourceRef=sourceRef,
artifacts=artifacts)
# Create the Alert
id = None
response = api.create_alert(alert)
if response.status_code == 201:
if args.verbose:
print('[INFO] Created alert %s' % response.json()['sourceRef'])
else:
print('[ERROR] Cannot create alert: %s (%s)' %
(response.status_code, response.text))
return False
else:
# Prepare the sample case
tasks = []
for task in config['caseTasks']:
tasks.append(CaseTask(title=task))
# Prepare the custom fields
customFields = CustomFieldHelper() \
.add_string('from', fromField) \
.add_string('attachment', str(attachments)) \
.build()
# If a case template is specified, use it instead of the tasks
m = 1
if m == 1:
templates = []
for task in config['caseTemplates']:
templates.append(task)
temptouse = config['caseTemplate']
descrip = re.compile('-"(.+)"')
name = re.compile('(.+)-"')
for x in templates:
z = descrip.search(x)
tempVar = name.search(x)
searchVar = z.group(1)
tempVar = tempVar.group(1)
if searchVar in subjectField:
print(
x
) #if 2 template names in subject, take the latest defined
temptouse = tempVar
print("TEMPLATE", temptouse)
if body:
testerVar = False
print("body")
try:
albert = re.compile('Albert Incident #: (\d+)')
m = albert.search(body)
albertId = m.group(1)
print(albertId)
customFields = CustomFieldHelper() \
.add_string('from', fromField) \
.add_string('attachment', str(attachments)) \
.add_string('albertId', albertId) \
.build()
print(customFields)
except:
print("albert id doesnt exist")
if "Update" in subjectField: #update code
testerVar = True
print("UPDATE")
#INTIAL
try:
findBodyInfo = re.compile('---((?:.+[\r\n]+)+)---')
except:
print(
"Unable to update, unable to find two '---'s, exiting.."
)
sys.exit(0)
m = findBodyInfo.search(body)
bigGroup = m.group(1)
caseId = parseBody("Case Id", bigGroup)
print("caseid", caseId)
try:
caseId = int(caseId)
except:
print("invalid case id")
updateACase(caseId, bigGroup, fromField, attachments)
id = None
if testerVar == True:
print("g")
sys.exit(0)
#end update code
caseTags = []
for tag in config['caseTags']:
descripFound = descrip.search(tag)
nameFound = name.search(tag)
descripFound = descripFound.group(1)
nameFound = nameFound.group(1)
if descripFound == 'always':
caseTags.append(nameFound)
elif descripFound in bodyMessage:
caseTags.append(nameFound)
try:
#
# Add observables found in the mail body
#
artifacts = []
if config['thehiveObservables'] and len(observables) > 0:
for o in observables:
artifacts.append(
AlertArtifact(dataType=o['type'], data=o['value']))
#
# Prepare tags - add alert keywords found to the list of tags
#
tags = config['alertTags']
match = re.findall(config['alertKeywords'], subjectField)
for m in match:
tags.append(m)
#
# Prepare the alert
#
sourceRef = str(uuid.uuid4())[0:6]
alert = Alert(
title=subjectField,
tlp=int(
config['alertTLP']
), #setting it blank since custom template allows default color, set it back to tlp = int for conf value
tags=caseTags,
description=body,
type='external',
source=fromField,
sourceRef=sourceRef,
customFields=customFields,
severity=None,
artifacts=artifacts)
except FileExistsError:
print(
"Error with creating alert, wrong template name or tags?")
else:
print("")
# Create the alert
response = api.create_alert(alert)
print("Alert being created..")
if response.status_code == 201:
newID = response.json()['id']
if args.verbose:
print('[INFO] Created alert %s' % response.json()['caseId'])
if len(attachments) > 0:
for path in attachments:
observable = CaseObservable(
dataType='file',
data=[path],
tlp=int(config['caseTLP']),
ioc=False,
tags=config['caseTags'],
message='Found as email attachment')
response = api.create_case_observable(newID, observable)
if response.status_code == 201:
if args.verbose:
print('[INFO] Added observable %s to case ID %s' %
(path, newID))
os.unlink(path)
else:
print('[WARNING] Cannot add observable: %s - %s (%s)' %
(path, response.status_code, response.text))
#
# Add observables found in the mail body
#
if config['thehiveObservables'] and len(observables) > 0:
for o in observables:
observable = CaseObservable(
dataType=o['type'],
data=o['value'],
tlp=int(config['caseTLP']),
ioc=False,
tags=caseTags, #switched to custom tags
message='Found in the email body')
response = api.create_case_observable(newID, observable)
if response.status_code == 201:
if args.verbose:
print(
'[INFO] Added observable %s: %s to case ID %s'
% (o['type'], o['value'], newID))
else:
print(
'[WARNING] Cannot add observable %s: %s - %s (%s)'
% (o['type'], o['value'], response.status_code,
response.text))
else:
print('[ERROR] Cannot create case: %s (%s)' %
(response.status_code, response.text))
return False
return True
def submitTheHive(message):
'''Create a new case in TheHive based on the email'''
# Decode email
msg = email.message_from_bytes(message)
decode = email.header.decode_header(msg['From'])[0]
fromField = str(decode[0])
decode = email.header.decode_header(msg['Subject'])[0]
subjectField = str(decode[0])
if args.verbose:
print("[INFO] From: %s Subject: %s" % (fromField, subjectField))
attachments = []
body = ''
for part in msg.walk():
if part.get_content_type() == "text/plain":
body = part.get_payload(decode=True).decode()
else:
filename = part.get_filename()
if filename:
print("[INFO] Found attachment: %s" % filename)
attachments.append(filename)
api = TheHiveApi(config['thehiveURL'], config['thehiveUser'],
config['thehivePassword'], {
'http': '',
'https': ''
})
if '[ALERT]' in subjectField:
# Prepare the alert
sourceRef = str(uuid.uuid4())[0:6]
alert = Alert(title=subjectField.replace('[ALERT]', ''),
tlp=int(config['alertTLP']),
tags=config['alertTags'],
description=body,
type='external',
source=fromField,
sourceRef=sourceRef)
# Create the Alert
id = None
response = api.create_alert(alert)
if response.status_code == 201:
if args.verbose:
print('[INFO] Created alert %s' % response.json()['sourceRef'])
else:
print('[ERROR] Cannot create alert: %s (%s)' %
(response.status_code, response.text))
sys.exit(0)
else:
# Prepare the sample case
tasks = []
for task in config['caseTasks']:
tasks.append(CaseTask(title=task))
# Prepare the custom fields
customFields = CustomFieldHelper()\
.add_string('from', fromField)\
.add_string('attachment', str(attachments))\
.build()
case = Case(title=subjectField,
tlp=int(config['caseTLP']),
flag=False,
tags=config['caseTags'],
description=body,
tasks=tasks,
customFields=customFields)
# Create the case
id = None
response = api.create_case(case)
if response.status_code == 201:
if args.verbose:
print('[INFO] Created case %s' % response.json()['caseId'])
else:
print('[ERROR] Cannot create case: %s (%s)' %
(response.status_code, response.text))
sys.exit(0)
return
def export_inc_to_alert(settings, api, incident, events, rec_list):
descript = ('#### Cсылка в MP SIEM: **' + settings['export_core_url'] +
'/#/incident/incidents/view/' + incident["id"] +
'?groupId=all_incidents&tabName=tasks**\n\n\n')
if events is not None:
event_counter = 0
artifacts = []
# print(json.dumps(events, indent=4, sort_keys=True))
try:
for event in events:
event_norm = get_event(event["id"], settings)
# print(json.dumps(event, indent=4, sort_keys=True))
event_counter += 1
descript += ('### Cобытие ' + str(event_counter) + ': **' +
event_norm["id"] + '**\n\n' +
'|Поле | Значение |\n' +
'|---------------|:-------------|\n')
try:
if event_norm["correlation_type"]:
title_alert = event_norm["text"]
except KeyError:
pass
observables_dict = {
'subject.name': 'other',
'object.name': 'other',
'src.ip': 'ip',
'dst.ip': 'ip',
'src.hostname': 'fqdn',
'dst.hostname': 'fqdn',
'event_src.host': 'fqdn'
}
for even in event_norm:
if str(even) != '_meta':
descript += '| **' + str(even) + '**| ' + str(event_norm[even]).replace("\n", "\\n"). \
replace("\r", "\\r").replace("|", "**OR**") + '|\n'
try:
artifacts.append(
AlertArtifact(
dataType=observables_dict[str(even)],
data=event_norm[even]))
except:
pass
# Prepare the custom fields
customFields = CustomFieldHelper() \
.add_string('category', incident["category"]) \
.add_string('type', incident["type"]) \
.build()
# Словарь соответствия Severity в MP SIEM и The HIVE
severity_dict = {'Low': 1, 'Medium': 2, 'High': 3}
alert = Alert(
title=title_alert + ' (' + str(event_counter) + ' Events)',
tlp=3,
tags=[
incident["name"], incident["category"], incident["type"]
],
description=descript,
type=incident["type"],
source='MP SIEM',
date=(incident["created"] * 1000),
sourceRef=str(incident["key"]),
severity=severity_dict[incident["severity"]],
artifacts=artifacts,
customFields=customFields)
if event_counter > 1:
response = api.create_alert(alert)
else:
update_time_from()
sys.exit(0)
if response.status_code == 201:
# if settings['debug'] == 1:
# print(json.dumps(response.json(), indent=4, sort_keys=True))
print_log("INFO: Export incident " + incident["key"] +
" to HIVE alert **" + incident["name"] + "**")
write_incident_file(settings['logfile'], rec_list[-1])
else:
print_log('ERROR: {}/{}'.format(response.status_code,
response.text))
update_time_from()
if response.status_code == 500:
write_incident_file(settings['logfile'], rec_list[-1])
sys.exit(0)
except:
print_log("WARNING: Export incident FAILED. Waiting.")
update_time_from()
sys.exit(0)
else:
print_log("WARNING: Incident without events. Waiting.")
update_time_from()
sys.exit(0)
def parse_hooks(self):
"""
Check for new MISP Alert containing supported IOC to search automatically
"""
if self.webhook.isNewMispAlert():
logger.info(
'Alert {} has been tagged as MISP and is just created'.format(
self.webhook.data['rootId']))
#Check alert for supported ioc types
supported_iocs = False
for artifact in self.webhook.data['object']['artifacts']:
if artifact['dataType'] in self.qr_config[
'supported_datatypes']:
supported_iocs = True
#Promote alert to case if there are support ioc types
if supported_iocs:
alert_id = self.webhook.data['rootId']
casetemplate = "MISP Event"
logger.info('Alert {} contains IOCs that are supported'.format(
alert_id))
response = self.TheHiveConnector.createCaseFromAlert(
alert_id, casetemplate)
self.report_action = 'createCase'
"""
Add timestamps to keep track of the search activity per case (we do not want to keep searching forever)
"""
#Perform automated Analyzer runs for supported observables in a case that has been created from a MISP alert
if self.webhook.isNewMispCase():
logger.info(
'Case {} has been tagged as MISP and is just created'.format(
self.webhook.data['rootId']))
#Retrieve caseid
caseid = self.webhook.data['object']['id']
#Add customFields firstSearched and lastSearched
#Create a Case object? Or whatever it is
case = Case()
#Add the case id to the object
case.id = caseid
#Debug output
logger.info('Updating case %s' % case.id)
#Define which fields need to get updated
fields = ['customFields']
#Retrieve all required attributes from the alert and add them as custom fields to the case
current_time = int(round(time.time() * 1000))
customFields = CustomFieldHelper()\
.add_date('firstSearched', current_time)\
.add_date('lastSearched', current_time)\
.build()
#Add custom fields to the case object
case.customFields = customFields
#Update the case
self.TheHiveConnector.updateCase(case, fields)
self.report_action = 'updateCase'
"""
Start the analyzers automatically for MISP observables that are supported and update the case with a new timestamp
"""
#Automatically run Analyzers for newly created MISP cases where supported IOC's are present
if self.webhook.isNewMispArtifact():
logger.info(
'Case artifact is tagged with "MISP-extern". Checking if observable is of a supported type'
)
#Retrieve caseid
caseid = self.webhook.data['rootId']
#Retrieve case data
case_data = self.TheHiveConnector.getCase(caseid)
#List all supported ioc's for the case
observable = self.webhook.data['object']
#When supported, start a cortex analyzer for it
if observable['dataType'] in self.qr_config['supported_datatypes']:
supported_observable = observable['_id']
#Trigger a search for the supported ioc
logger.info('Launching analyzers for observable: {}'.format(
observable['_id']))
response = self.CortexConnector.runAnalyzer(
"Cortex-intern", supported_observable,
"IBMQRadar_Search_Manual_0_1")
#Add customFields firstSearched and lastSearched
#Create a Case object
case = Case()
#Add the case id to the object
case.id = caseid
#Debug output
logger.info('Updating case %s' % case.id)
#Define which fields need to get updated
fields = ['customFields']
#Retrieve all required attributes from the alert and add them as custom fields to the case
current_time = int(round(time.time() * 1000))
customFields = CustomFieldHelper()\
.add_date('firstSearched', case_data['customFields']['firstSearched']['date'])\
.add_date('lastSearched', current_time)\
.build()
#Add custom fields to the case object
case.customFields = customFields
#Update the case
self.TheHiveConnector.updateCase(case, fields)
self.report_action = 'updateCase'
"""
Automatically create a task for a found IOC
"""
#If the Job result contains a successful search with minimum of 1 hit, create a task to investigate the results
if self.webhook.isCaseArtifactJob() and self.webhook.isSuccess(
) and self.webhook.isMisp():
#Case ID
caseid = self.webhook.data['rootId']
#Load Case information
case_data = self.TheHiveConnector.getCase(caseid)
logger.info(
'Job {} is part of a case that has been tagged as MISP case and has just finished'
.format(self.webhook.data['object']['cortexJobId']))
#Check if the result count higher than 0
if int(
float(self.webhook.data['object']['report']['summary']
['taxonomies'][0]['value'])) > 0:
logger.info(
'Job {} contains hits, checking if a task is already present for this observable'
.format(self.webhook.data['object']['cortexJobId']))
#Retrieve case task information
response = self.TheHiveConnector.getCaseTasks(caseid)
case_tasks = response.json()
#Load CaseTask template
casetask = CaseTask()
#Observable + Link
observable = self.webhook.data['object']['artifactId']
observable_link = TheHive.get(
'url'
) + "/index.html#!/case/" + caseid + "/observables/" + self.webhook.data[
'object']['artifactId']
#Task name
casetask.title = "Investigate found IOC with id: {}".format(
observable)
#Date
date_found = time.strftime("%d-%m-%Y %H:%M")
case_task_found = False
for case_task in case_tasks:
#Check if task is present for investigating the new results
if casetask.title == case_task['title']:
case_task_found = True
if not case_task_found:
logger.info(
'No task found, creating task for observable found in job {}'
.format(self.webhook.data['object']['cortexJobId']))
#Add description
casetask.description = "The following ioc is hit in the environment. Investigate the results and act accordingly:\n\n"
casetask.description = casetask.description + "{} is seen on {}\n".format(
observable_link, date_found)
#Check if case is closed
if case_data['status'] == "Resolved":
#Create a Case object? Or whatever it is
case = Case()
#Add the case id to the object
case.id = caseid
logger.info('Updating case %s' % case.id)
#Define which fields need to get updated
fields = ['status']
#Reopen the case
case.status = "Open"
#Update the case
self.TheHiveConnector.updateCase(case, fields)
#Add the case task
self.TheHiveConnector.createTask(caseid, casetask)
self.report_action = 'createTask'
def submitTheHive(emailFilePath):
'''
Create a new case in TheHive based on the email
Return 'TRUE' is successfully processed otherwise 'FALSE'
'''
global log
mailName, mailType = os.path.splitext(emailFilePath)
if mailType == ".msg":
fromField, subjectField, observables, body, attachments = readMsg(emailFilePath)
elif mailType == ".eml":
fromField, subjectField, observables, body, attachments = readEml(emailFilePath)
# Cleanup observables (remove duplicates)
new_observables = []
for o in observables:
if not {'type': o['type'], 'value': o['value'] } in new_observables:
# Is the observable whitelisted?
if isWhitelisted(o['value']):
log.debug('Skipping whitelisted observable: %s' % o['value'])
else:
new_observables.append({ 'type': o['type'], 'value': o['value'] })
log.debug('Found observable %s: %s' % (o['type'], o['value']))
else:
log.info('Ignoring duplicate observable: %s' % o['value'])
log.info("Removed duplicate observables: %d -> %d" % (len(observables), len(new_observables)))
observables = new_observables
api = TheHiveApi(config['thehiveURL'], config['thehiveApiKey'])
# Search for interesting keywords in subjectField:
log.debug("Searching for %s in '%s'" % (config['alertKeywords'], subjectField))
if re.match(config['alertKeywords'], subjectField, flags=0):
#
# Add observables found in the mail body
#
artifacts = []
if config['thehiveObservables'] and len(observables) > 0:
for o in observables:
artifacts.append(AlertArtifact(dataType=o['type'], data=o['value']))
#
# Prepare tags - add alert keywords found to the list of tags
#
tags = list(config['alertTags'])
match = re.findall(config['alertKeywords'], subjectField)
for m in match:
tags.append(m)
#
# Prepare the alert
#
sourceRef = str(uuid.uuid4())[0:6]
alert = Alert(title=subjectField.replace('[ALERT]', ''),
tlp = int(config['alertTLP']),
tags = tags,
description = body,
type = 'external',
source = fromField,
sourceRef = sourceRef,
artifacts = artifacts)
# Create the Alert
id = None
response = api.create_alert(alert)
if response.status_code == 201:
log.info('Created alert %s' % response.json()['sourceRef'])
else:
log.error('Cannot create alert: %s (%s)' % (response.status_code, response.text))
return False
else:
# Prepare the sample case
tasks = []
for task in config['caseTasks']:
tasks.append(CaseTask(title=task))
# Prepare the custom fields
customFields = CustomFieldHelper()\
.add_string('from', fromField)\
.add_string('attachment', str(attachments))\
.build()
# If a case template is specified, use it instead of the tasks
if len(config['caseTemplate']) > 0:
case = Case(title=subjectField,
tlp = int(config['caseTLP']),
flag = False,
tags = config['caseTags'],
description = body,
template = config['caseTemplate'],
customFields = customFields)
else:
# Define variables
HIVE_SERVER_IP = configuration.SYSTEM_SETTINGS['HIVE_SERVER_IP']
HIVE_SERVER_PORT = configuration.SYSTEM_SETTINGS['HIVE_SERVER_PORT']
HIVE_API_KEY = configuration.SYSTEM_SETTINGS['HIVE_API_KEY']
api = TheHiveApi("http://%s:%s" % (HIVE_SERVER_IP, HIVE_SERVER_PORT), HIVE_API_KEY)
# Prepare observables
artifacts = [
AlertArtifact(dataType='ip', data='8.8.8.8'),
AlertArtifact(dataType='domain', data='google.com'),
AlertArtifact(dataType='domain', data='test'),
AlertArtifact(dataType='domain', data='sample.txt', sighted=True, ioc=True)
]
# Prepare custom fields
customFields = CustomFieldHelper()\
.build()
# Prepare the sample Alert
sourceRef = str(uuid.uuid4())[0:6]
alert = Alert(title='Tier3 Test',
tlp=3,
severity=3,
tags=['TheHive4Py', 'sample'],
description='N/A',
type='external',
source='instance1',
sourceRef=sourceRef,
artifacts=artifacts,
customFields=customFields)
# Create the alert
def doWork(self):
#Build a loop that keeps the thread alive until queue is empty
while not q.empty():
#Build up the threads
thread_count = threading.active_count()
#Make sure that the thread count is lower than configured limit and is lower than the queue size
if thread_count < concurrent and thread_count < q.qsize():
new_thread_count = thread_count + 1
logger.info(
'Current queue size(%i) allows more threads. Creating additional thread: %i'
% (q.qsize(), new_thread_count))
t = Thread(target=self.doWork)
t.daemon = True
t.start()
logger.debug('Created thread: %i' % new_thread_count)
#Retrieve a queued item
queued_item = q.get()
#Handle a queued item based on its provided action
if queued_item['action'] == "search_observables":
logger.info('Working on %s from queue, caseid: %s' %
(queued_item['action'], queued_item['data']['id']))
case_data = queued_item['data']
logger.debug("event: %s" % case_data)
#Store CaseID
caseid = case_data['id']
#If the case is within scope of the search range. Perform the search
#if (case_data['customFields']['lastSearched']['date'] - case_data['customFields']['firstSearched']['date']) < 5184000000:
#logger.info('Observables in case {} have not yet been searched for longer than two months. Starting analyzers'.format(case_data['id']))
self.response = self.thapi.get_case_observables(caseid)
#Perform a search for ioc's per case in the RS search results (Trigger Cortex Analyzer)
for observable in self.response.json():
searched_for = False
logger.debug("observable: %s" % observable)
logger.debug("current_time %s, observable_time %s" %
(self.current_time, observable['startDate']))
#Check if observables are not older than 2 months or 6 months for TLP:RED
if (((self.current_time - observable['startDate']) <
5184000000)
or (observable['tlp'] == 3 and
((self.current_time - observable['startDate'])
< 15552000))):
self.searchtype = observable['dataType']
if self.searchtype in self.qr_enabled_datatypes:
self.supported_observable = observable['_id']
#Trigger a search for the supported ioc
logger.info(
'Launching analyzers for observable: {}'.
format(self.supported_observable))
self.response = self.thapi.run_analyzer(
"Cortex-intern", self.supported_observable,
"IBMQRadar_Search_Automated_0_1")
self.check_response(self.response)
searched_for = True
if searched_for:
#Add customFields firstSearched and lastSearched
#Create a Case object? Or whatever it is
self.case = Case()
#Add the case id to the object
self.case.id = caseid
#Debug output
logger.info('Updating case %s' % self.case.id)
#Define which fields need to get updated
fields = ['customFields']
#Retrieve all required attributes from the alert and add them as custom fields to the case
self.customFields = CustomFieldHelper()\
.add_date('firstSearched', case_data['customFields']['firstSearched']['date'])\
.add_date('lastSearched', self.current_time)\
.build()
#Add custom fields to the case object
self.case.customFields = self.customFields
#Update the case
self.response = self.thapi.update_case(self.case, fields)
self.check_response(self.response)
logger.info("Queue is empty, nothing left to do")