class Automators(Main):
def __init__(self, cfg, use_case_config):
self.logger = logging.getLogger(__name__)
self.logger.info('Initiating QRadar Automators')
self.cfg = cfg
self.use_case_config = use_case_config
self.TheHiveConnector = TheHiveConnector(cfg)
self.TheHiveAutomators = TheHiveAutomators(cfg, use_case_config)
self.QRadarConnector = QRadarConnector(cfg)
def search(self, action_config, webhook):
# Only continue if the right webhook is triggered
self.logger.debug("action_config:{}".format(action_config))
if webhook.isImportedAlert():
pass
else:
return False
# Define variables and actions based on certain webhook types
self.case_id = webhook.data['object']['case']
self.logger.debug(self.case_id)
self.enriched = False
for query_name, query_config in action_config.items():
try:
self.logger.debug('Found the following query: {}'.format(
query_config['query']))
self.query_variables = {}
self.query_variables['input'] = {}
# Render query
try:
# Prepare the template
self.template = Template(query_config['query'])
# Find variables in the template
self.template_env = Environment()
self.template_parsed = self.template_env.parse(
query_config['query'])
# Grab all the variales from the template and try to find them in the description
self.template_vars = meta.find_undeclared_variables(
self.template_parsed)
self.logger.debug(
"Found the following variables in query: {}".format(
self.template_vars))
for template_var in self.template_vars:
# Skip dynamically generated Stop_time variable
if template_var == "Stop_Time":
continue
self.logger.debug(
"Looking up variable required for template: {}".
format(template_var))
# Replace the underscore from the variable name to a white space as this is used in the description table
self.template_var_with_ws = template_var.replace(
"_", " ")
self.case_data = self.TheHiveConnector.getCase(
self.case_id)
self.logger.debug('output for get_case: {}'.format(
self.case_data))
self.query_variables['input'][
template_var] = self.TheHiveAutomators.fetchValueFromMDTable(
self.case_data['description'],
self.template_var_with_ws)
if 'Start_Time' not in self.query_variables['input']:
self.logger.warning(
"Could not find Start Time value required to build the search"
)
# Parse times required for the query (with or without offset)
if template_var == "Start_Time":
self.logger.debug(
"Found Start Time: %s" %
self.query_variables['input']['Start_Time'])
if 'start_time_offset' in query_config:
self.query_variables['input'][
'Start_Time'] = self.parseTimeOffset(
self.query_variables['input']
['Start_Time'],
self.cfg.get(
'Automation',
'event_start_time_format'),
query_config['start_time_offset'],
self.cfg.get('QRadar', 'time_format'))
else:
self.query_variables['input'][
'Start_Time'] = self.query_variables[
'input']['Start_Time']
if 'stop_time_offset' in query_config:
self.query_variables['input'][
'Stop_Time'] = self.parseTimeOffset(
self.query_variables['input']
['Start_Time'],
self.cfg.get(
'Automation',
'event_start_time_format'),
query_config['stop_time_offset'],
self.cfg.get('QRadar', 'time_format'))
else:
self.query_variables['input'][
'Stop_Time'] = datetime.now().strftime(
self.cfg.get(
'Automation',
'event_start_time_format'))
self.rendered_query = self.template.render(
self.query_variables['input'])
self.logger.debug("Rendered the following query: %s" %
self.rendered_query)
except Exception as e:
self.logger.warning(
"Could not render query due to missing variables",
exc_info=True)
continue
# Perform search queries
try:
self.rendered_query_result = self.QRadarConnector.aqlSearch(
self.rendered_query)
# Check results
self.logger.debug(
'The search result returned the following information: \n %s'
% self.rendered_query_result)
except Exception as e:
self.logger.warning("Could not perform query",
exc_info=True)
continue
try:
if query_config['create_thehive_task']:
self.logger.debug("create task is enabled")
# Task name
self.uc_task_title = query_config['thehive_task_title']
self.uc_task_description = "The following information is found. Investigate the results and act accordingly:\n\n\n\n"
# create a table header
self.table_header = "|"
self.rows = "|"
if len(self.rendered_query_result['events']) != 0:
for key in self.rendered_query_result['events'][
0].keys():
self.table_header = self.table_header + " %s |" % key
self.rows = self.rows + "---|"
self.table_header = self.table_header + "\n" + self.rows + "\n"
self.uc_task_description = self.uc_task_description + self.table_header
# Create the data table for the results
for event in self.rendered_query_result['events']:
self.table_data_row = "|"
for field_key, field_value in event.items():
# Escape pipe signs
if field_value:
field_value = field_value.replace(
"|", "|")
# Use to create some additional spacing
self.table_data_row = self.table_data_row + " %s |" % field_value
self.table_data_row = self.table_data_row + "\n"
self.uc_task_description = self.uc_task_description + self.table_data_row
else:
self.uc_task_description = self.uc_task_description + "No results \n"
# Add the case task
self.uc_task = self.TheHiveAutomators.craftUcTask(
self.uc_task_title, self.uc_task_description)
self.TheHiveConnector.createTask(
self.case_id, self.uc_task)
except Exception as e:
self.logger.debug(e)
pass
try:
if query_config['create_ioc']:
self.logger.debug("create IOC is enabled")
self.comment = "offense enrichment"
#static tags list
self.tags = ['synapse']
#want to add SECID of the rule as well in the tag
rule_secid = [
x for x in webhook.data['object']['tags']
if x.startswith('SEC')
]
self.tags.extend(rule_secid)
self.uc_ioc_type = query_config['ioc_type']
if len(self.rendered_query_result['events']) != 0:
for event in self.rendered_query_result['events']:
for field_key, field_value in event.items():
self.TheHiveConnector.addObservable(
self.case_id, self.uc_ioc_type,
list(field_value.split(",")),
self.tags, self.comment)
except Exception as e:
self.logger.debug(e)
pass
except Exception as e:
self.logger.debug(
'Could not process the following query: {}\n{}'.format(
query_config, e))
continue
# Return True when succesful
return True
def enrichAlert(self, action_config, webhook):
# Only continue if the right webhook is triggered
if webhook.isNewAlert():
pass
else:
return False
# Define variables and actions based on certain webhook types
# Alerts
self.alert_id = webhook.data['object']['id']
self.alert_description = webhook.data['object']['description']
self.query_variables = {}
self.query_variables['input'] = {}
self.enriched = False
# Prepare search queries for searches
for query_name, query_config in action_config.items():
try:
self.logger.info('Found the following query: %s' %
(query_name))
self.query_variables[query_name] = {}
# Render query
try:
# Prepare the template
self.template = Template(query_config['query'])
# Find variables in the template
self.template_env = Environment()
self.template_parsed = self.template_env.parse(
query_config['query'])
# Grab all the variales from the template and try to find them in the description
self.template_vars = meta.find_undeclared_variables(
self.template_parsed)
self.logger.debug(
"Found the following variables in query: {}".format(
self.template_vars))
for template_var in self.template_vars:
# Skip dynamically generated Stop_time variable
if template_var == "Stop_Time":
continue
self.logger.debug(
"Looking up variable required for template: {}".
format(template_var))
# Replace the underscore from the variable name to a white space as this is used in the description table
self.template_var_with_ws = template_var.replace(
"_", " ")
self.alert_data = self.TheHiveConnector.getAlert(
self.alert_id)
self.logger.debug('output for get_alert: {}'.format(
self.alert_data))
self.query_variables['input'][
template_var] = self.TheHiveAutomators.fetchValueFromMDTable(
self.alert_data['description'],
self.template_var_with_ws)
# Parse times required for the query (with or without offset)
if template_var == "Start_Time":
self.logger.debug(
"Found Start Time: %s" %
self.query_variables['input']['Start_Time'])
if 'start_time_offset' in query_config:
self.query_variables['input'][
'Start_Time'] = self.parseTimeOffset(
self.query_variables['input']
['Start_Time'],
self.cfg.get(
'Automation',
'event_start_time_format'),
query_config['start_time_offset'],
self.cfg.get('QRadar', 'time_format'))
else:
self.query_variables['input'][
'Start_Time'] = self.query_variables[
'input']['Start_Time']
if 'stop_time_offset' in query_config:
self.query_variables['input'][
'Stop_Time'] = self.parseTimeOffset(
self.query_variables['input']
['Start_Time'],
self.cfg.get(
'Automation',
'event_start_time_format'),
query_config['stop_time_offset'],
self.cfg.get('QRadar', 'time_format'))
else:
self.query_variables['input'][
'Stop_Time'] = datetime.now().strftime(
self.cfg.get(
'Automation',
'event_start_time_format'))
if not self.query_variables['input']['Start_Time']:
self.logger.warning("Could not find Start Time value ")
raise GetOutOfLoop
self.query_variables[query_name][
'query'] = self.template.render(
self.query_variables['input'])
self.logger.debug(
"Rendered the following query: %s" %
self.query_variables[query_name]['query'])
except Exception as e:
self.logger.warning(
"Could not render query due to missing variables",
exc_info=True)
raise GetOutOfLoop
# Perform search queries
try:
self.query_variables[query_name][
'result'] = self.QRadarConnector.aqlSearch(
self.query_variables[query_name]['query'])
except Exception as e:
self.logger.warning("Could not perform query",
exc_info=True)
raise GetOutOfLoop
# Check results
self.logger.debug(
'The search result returned the following information: \n %s'
% self.query_variables[query_name]['result'])
# making enrichment results presentable
clean_enrichment_results = self.TheHiveAutomators.make_it_presentable(
self.query_variables[query_name]['result']['events'][0]
['enrichment_result'])
# Add results to description
success = self.enrichAlertDescription(
self.alert_data['description'], query_name,
self.query_variables[query_name]['result']['events'][0]
['enrichment_result'])
if not success:
self.logger.warning(
"Could not add results from the query to the description. Error: {}"
.format(e))
raise GetOutOfLoop
except GetOutOfLoop:
pass
return True
class Automators(Main):
def __init__(self, cfg, use_case_config):
self.logger = logging.getLogger(__name__)
self.logger.info('Initiating QRadar Automators')
self.cfg = cfg
self.use_case_config = use_case_config
self.TheHiveConnector = TheHiveConnector(cfg)
self.TheHiveAutomators = TheHiveAutomators(cfg, use_case_config)
self.QRadarConnector = QRadarConnector(cfg)
def checkSiem(self, action_config, webhook):
#Only continue if the right webhook is triggered
if webhook.isImportedAlert() or webhook.isNewAlert() or webhook.isQRadarAlertUpdateFollowTrue():
pass
else:
return False
#Define variables and actions based on certain webhook types
#Alerts
if webhook.isNewAlert() or webhook.isQRadarAlertUpdateFollowTrue():
self.alert_id = webhook.data['object']['id']
self.alert_description = webhook.data['object']['description']
self.supported_query_type = 'enrichment_queries'
#Cases
elif webhook.isImportedAlert():
self.case_id = webhook.data['object']['case']
self.supported_query_type = 'search_queries'
self.query_variables = {}
self.query_variables['input'] = {}
self.enriched = False
#Prepare search queries for searches
for query_name, query_config in action_config[self.supported_query_type].items():
try:
self.logger.info('Found the following query: %s' % (query_name))
self.query_variables[query_name] = {}
#Render query
try:
#Prepare the template
self.template = Template(query_config['query'])
#Find variables in the template
self.template_env = Environment()
self.template_parsed = self.template_env.parse(query_config['query'])
#Grab all the variales from the template and try to find them in the description
self.template_vars = meta.find_undeclared_variables(self.template_parsed)
self.logger.debug("Found the following variables in query: {}".format(self.template_vars))
for template_var in self.template_vars:
#Skip dynamically generated Stop_time variable
if template_var == "Stop_Time":
continue
self.logger.debug("Looking up variable required for template: {}".format(template_var))
#Replace the underscore from the variable name to a white space as this is used in the description table
self.template_var_with_ws = template_var.replace("_", " ")
self.query_variables['input'][template_var] = self.TheHiveAutomators.fetchValueFromDescription(webhook,self.template_var_with_ws)
#Parse times required for the query (with or without offset)
if template_var == "Start_Time":
self.logger.debug("Found Start Time: %s" % self.query_variables['input']['Start_Time'])
if 'start_time_offset' in query_config:
self.query_variables['input']['Start_Time'] = self.parseTimeOffset(self.query_variables['input']['Start_Time'], self.cfg.get('Automation', 'event_start_time_format'), query_config['start_time_offset'], self.cfg.get('QRadar', 'time_format'))
else:
self.query_variables['input']['Start_Time'] = self.query_variables['input']['Start_Time']
if 'stop_time_offset' in query_config:
self.query_variables['input']['Stop_Time'] = self.parseTimeOffset(self.query_variables['input']['Start_Time'], self.cfg.get('Automation', 'event_start_time_format'), query_config['stop_time_offset'], self.cfg.get('QRadar', 'time_format'))
else:
self.query_variables['input']['Stop_Time'] = datetime.now().strftime(self.cfg.get('Automation', 'event_start_time_format'))
if not self.query_variables['input']['Start_Time']:
self.logger.warning("Could not find Start Time value ")
raise GetOutOfLoop
self.query_variables[query_name]['query'] = self.template.render(self.query_variables['input'])
self.logger.debug("Rendered the following query: %s" % self.query_variables[query_name]['query'])
except Exception as e:
self.logger.warning("Could not render query due to missing variables", exc_info=True)
raise GetOutOfLoop
#Perform search queries
try:
self.query_variables[query_name]['result'] = self.QRadarConnector.aqlSearch(self.query_variables[query_name]['query'])
except Exception as e:
self.logger.warning("Could not perform query", exc_info=True)
raise GetOutOfLoop
#Check results
self.logger.debug('The search result returned the following information: \n %s' % self.query_variables[query_name]['result'])
if self.supported_query_type == "search_queries":
#Task name
self.uc_task_title = query_config['task_title']
self.uc_task_description = "The following information is found. Investigate the results and act accordingly:\n\n\n\n"
#create a table header
self.table_header = "|"
self.rows = "|"
if len(self.query_variables[query_name]['result']['events']) != 0:
for key in self.query_variables[query_name]['result']['events'][0].keys():
self.table_header = self.table_header + " %s |" % key
self.rows = self.rows + "---|"
self.table_header = self.table_header + "\n" + self.rows + "\n"
self.uc_task_description = self.uc_task_description + self.table_header
#Create the data table for the results
for event in self.query_variables[query_name]['result']['events']:
self.table_data_row = "|"
for field_key, field_value in event.items():
# Escape pipe signs
if field_value:
field_value = field_value.replace("|", "|")
# Use to create some additional spacing
self.table_data_row = self.table_data_row + " %s |" % field_value
self.table_data_row = self.table_data_row + "\n"
self.uc_task_description = self.uc_task_description + self.table_data_row
else:
self.uc_task_description = self.uc_task_description + "No results \n"
#Add the case task
self.uc_task = self.TheHiveAutomators.craftUcTask(self.uc_task_title, self.uc_task_description)
self.TheHiveConnector.createTask(self.case_id, self.uc_task)
if self.supported_query_type == "enrichment_queries":
#Add results to description
try:
if self.TheHiveAutomators.fetchValueFromDescription(webhook,query_name) != self.query_variables[query_name]['result']['events'][0]['enrichment_result']:
self.regex_end_of_table = ' \|\\n\\n\\n'
self.end_of_table = ' |\n\n\n'
self.replacement_description = '|\n | **%s** | %s %s' % (query_name, self.query_variables[query_name]['result']['events'][0]['enrichment_result'], self.end_of_table)
self.th_alert_description = self.TheHiveConnector.getAlert(self.alert_id)['description']
self.alert_description = re.sub(self.regex_end_of_table, self.replacement_description, self.th_alert_description)
self.enriched = True
#Update Alert with the new description field
self.updated_alert = Alert
self.updated_alert.description = self.alert_description
self.TheHiveConnector.updateAlert(self.alert_id, self.updated_alert, ["description"])
except Exception as e:
self.logger.warning("Could not add results from the query to the description. Error: {}".format(e))
raise GetOutOfLoop
except GetOutOfLoop:
pass
return True
class Automator():
def __init__(self, webhook, cfg, automation_config, modules):
"""
Class constructor
:return: use case report
:rtype: API call
"""
self.logger = logging.getLogger(__name__)
self.logger.info('Initiating Siem Integration')
self.cfg = cfg
self.app_dir = os.path.dirname(os.path.abspath(__file__)) + "/.."
self.automation_config = automation_config
self.TheHiveConnector = TheHiveConnector(cfg)
self.webhook = webhook
self.modules = modules
if cfg.getboolean('Automation', 'enable_customer_list',
fallback=False):
self.logger.info('Loading Customer configuration')
# Load optional customer config
self.customer_cfg = ConfigParser(
converters={
'list': lambda x: [i.strip() for i in x.split(';')]
})
self.confPath = self.app_dir + '/conf/customers.conf'
try:
self.logger.debug('Loading configuration from %s' %
self.confPath)
self.customer_cfg.read(self.confPath)
self.customers = self.customer_cfg.sections()
self.logger.debug('Loaded configuration for %s' %
self.customers)
except Exception as e:
self.logger.error('%s', __name__, exc_info=True)
def check_automation(self):
self.logger.info(
'Start parsing use cases for the SIEM based alerts/cases')
self.ucTaskId = False
self.report_action = 'None'
if 'tags' in self.webhook.data['object']:
self.tags = self.webhook.data['object']['tags']
# Add tagging to webhooks that are missing tags
elif 'artifactId' in self.webhook.data[
'object'] and self.webhook.isCaseArtifactJob:
self.logger.debug(
'Found artifact id {} for webhook {}. Retrieving tags from there'
.format(self.webhook.data['object']['artifactId'],
self.webhook.id))
self.tags = self.TheHiveConnector.getCaseObservable(
self.webhook.data['object']['artifactId'])['tags']
else:
self.tags = []
self.logger.warning("no tags found for webhook {}".format(
self.webhook.id))
self.automation_regexes = self.cfg.get('Automation',
'automation_regexes',
fallback=None)
if not self.automation_regexes:
self.logger.error(
"Could not find any regexes to find tags for automation")
return self.report_action
self.automation_ids = self.automation_config['automation_ids']
# loop through tags to see if there is a use case present
for tag in self.tags:
for automation_regex in self.automation_regexes:
# The tag should match this regex otherwise it is no use case
try:
tag = re.search(automation_regex, tag).group(0)
except Exception:
self.logger.debug("Tag: %s is not matching the uc regex" %
tag)
continue
# check if use case that is provided, matches the case
if tag in self.automation_ids:
self.found_a_id = tag
# Try to retrieve the defined actions
self.use_case_actions = self.automation_ids[
self.found_a_id]['automation']
# perform actions defined for the use case
for action, action_config in self.use_case_actions.items():
# Give automator information regarding the webhook as some actions are limited to the state of the alert/case
self.logger.info(
'Found the following action for %s: %s, with task %s'
% (self.found_a_id, action, action_config['task']))
# Add support for multiple tasks, loop them 1 by 1
if 'tasks' in action_config:
for task in action_config['tasks']:
action_config['task'] = task
# Run actions through the automator
if self.Automate(action_config, self.webhook,
self.modules):
continue
else:
self.logger.info(
'Did not find any supported actions with details: task:{} tag:{} action:{}'
.format(action_config['task'],
self.found_a_id,
action_config['task']))
#Run actions through the automator
else:
if self.Automate(action_config, self.webhook,
self.modules):
continue
else:
self.logger.info(
'Fallback: Did not find any supported actions with details: task:{} tag:{} action:{}'
.format(action_config['task'],
self.found_a_id,
action_config['task']))
self.logger.info("Report action snapshot for the tag: {}".format(
self.report_action))
return self.report_action
def Automate(self, task_config, webhook, modules):
# Split the task name on the dot to have a module and a function variable in a list
try:
task = task_config['task'].split(".")
# Should probably also do some matching for words to mitigate some security concerns?
module_name = task[0]
function_name = task[1]
except Exception:
self.logger.error(
"{} does not seem to be a valid automator task name".format(
task),
exc_info=True)
return
try:
# Load the Automators class from the module to initialise it
automators = modules['automators'][module_name].Automators(
self.cfg, self.automation_config)
except KeyError as e:
self.logger.warning(
"Automator module not found: {}".format(module_name),
exc_info=True)
return False
try:
automator = getattr(automators, '{}'.format(function_name))
# Run the function for the task and return the results
results = automator(task_config, webhook)
# Return the results or True if the task was succesful without returning information
if results:
return results
else:
return False
except KeyError as e:
self.logger.warning("Automator task not found for {}: {}".format(
module_name, function_name),
exc_info=True)
return False
def enrichAlertDescription(self, alert_id, description, enrichment_key,
enrichment_value):
th_alert_description = self.TheHiveConnector.getAlert(
alert_id)['description']
# Split again, then parse the enrichment table part. After that the two splitted parts can be put together again
original_description, enrichment_table = retrieveSplittedDescription(
description)
# Primarily check if the split action worked. If the variables are the same, then the check if the key is found and does not contain the value already
if enrichment_table and self.fetchValueFromMDTable(
enrichment_table, enrichment_key) != enrichment_value:
regex_end_of_table = ' \|\\n\\n\\n'
end_of_table = ' |\n\n\n'
replacement_description = '|\n | **%s** | %s %s' % (
enrichment_key, enrichment_value, end_of_table)
alert_description = re.sub(regex_end_of_table,
replacement_description,
th_alert_description)
enriched = True
# Concat enrichment table with rest of description
# Update Alert with the new description field
updated_alert = Alert
updated_alert.description = alert_description
self.TheHiveConnector.updateAlert(alert_id, updated_alert,
["description"])